diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..311059aac --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,42 @@ +name: Helix acceptance benchmark + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + benchmark: + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Require the pinned public embedded wheel + run: python tools/check_helix_wheel.py + - name: Install isolated benchmark dependencies + run: | + python -m pip install -e . --no-deps + python -m pip install -r requirements-runtime.txt -r benchmarks/requirements-networkx.txt + - name: Run isolated NetworkX behavioral parity + env: + PYTHONPATH: . + run: python benchmarks/parity_networkx.py > parity-result.json + - name: Run 5k/15k and 20k/60k comparison + env: + PYTHONPATH: . + run: python benchmarks/helix_vs_networkx.py --out benchmark-result.json --check-gates + - uses: actions/upload-artifact@v4 + with: + name: helix-vs-networkx-${{ matrix.os }}-${{ github.sha }} + path: | + benchmark-result.json + parity-result.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 548b40f22..b0ac58293 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +name: Helix native CI on: push: @@ -18,10 +18,8 @@ jobs: steps: - uses: actions/checkout@v6 with: - # The audit-coverage, monolith-roundtrip, and always-on-roundtrip - # validators read blobs from origin/v8. A shallow checkout omits that - # ref, so fetch the full history here and the validators run for real - # (rather than skipping). The other jobs stay shallow. + # The heading audit reads a pinned pre-split v8 commit. A shallow + # checkout can omit it, so fetch full history for this job. fetch-depth: 0 - name: Install uv @@ -47,11 +45,23 @@ jobs: run: uv run --frozen python -m tools.skillgen --always-on-roundtrip test: - runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["3.10", "3.12"] - + include: + - os: ubuntu-latest + python-version: "3.10" + - os: ubuntu-latest + python-version: "3.12" + - os: ubuntu-24.04-arm + python-version: "3.12" + - os: macos-latest + python-version: "3.12" + - os: windows-latest + python-version: "3.10" + - os: windows-latest + python-version: "3.12" + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 with: @@ -65,6 +75,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Require the pinned public embedded wheel + run: python tools/check_helix_wheel.py + # --frozen installs straight from the committed uv.lock without re-resolving # or rewriting it, so CI never churns the lock. - name: Install dependencies @@ -77,14 +90,38 @@ jobs: run: | uv run --frozen graphify --help uv run --frozen graphify install + uv build + + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: astral-sh/setup-uv@v8.1.0 + with: + python-version: "3.12" + + - run: uv sync --all-extras --frozen + + - name: Ruff + run: uv run --frozen ruff check graphify tests tools benchmarks + + - name: Pyright native integration surface + run: >- + uv run --frozen pyright graphify/helix graphify/operations.py graphify/cli.py + graphify/cluster.py graphify/analyze.py graphify/benchmark.py + graphify/global_graph.py graphify/impact.py graphify/affected.py + graphify/paths.py graphify/prs.py graphify/export.py graphify/watch.py + + - name: Compile and whitespace checks + run: | + uv run --frozen python -m compileall -q graphify + git diff --check security-scan: # The dev deps include bandit and pip-audit. Run them in CI so a new # HIGH-severity finding or vulnerable dependency is caught on the PR that # introduces it, rather than at the next manual audit. - # Non-blocking for now (continue-on-error) to avoid breaking CI on - # pre-existing findings; remove continue-on-error after the initial - # cleanup pass. runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -98,9 +135,10 @@ jobs: run: uv sync --frozen - name: bandit (static security analysis) - continue-on-error: true run: uv run --frozen bandit -r graphify -ll - name: pip-audit (dependency vulnerabilities) - continue-on-error: true - run: uv run --frozen pip-audit --strict + run: | + uv export --quiet --frozen --all-extras --no-dev --no-emit-project \ + --output-file audited-requirements.txt + uv run --frozen pip-audit --strict --requirement audited-requirements.txt diff --git a/.github/workflows/release-graph.yml b/.github/workflows/release-graph.yml index 246ec40d9..b9b19d185 100644 --- a/.github/workflows/release-graph.yml +++ b/.github/workflows/release-graph.yml @@ -37,12 +37,12 @@ jobs: run: uv run --frozen graphify cluster-only . --no-label - name: Generate HTML viewer - run: uv run --frozen graphify export html --graph graphify-out/graph.json + run: uv run --frozen graphify export html --graph graphify-out/graph.helix - name: Bundle release asset run: | mkdir -p dist-graph - cp graphify-out/graph.json dist-graph/ + cp -R graphify-out/graph.helix dist-graph/ cp graphify-out/graph.html dist-graph/ [ -f graphify-out/GRAPH_REPORT.md ] && cp graphify-out/GRAPH_REPORT.md dist-graph/ || true tar -czf graphify-self-graph.tar.gz -C dist-graph . diff --git a/.gitignore b/.gitignore index 0a6775b2a..8674115f1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ build/ .ruff_cache/ *.so *.egg +.DS_Store .graphify/ graphify-out/ .graphify_*.json diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5672bf0df..4a1effe34 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -8,27 +8,53 @@ graphify is a Claude Code skill backed by a Python library. The skill orchestrat detect() → extract() → build_graph() → cluster() → analyze() → report() → export() ``` -Each stage is a single function in its own module. They communicate through plain Python dicts and NetworkX graphs - no shared state, no side effects outside `graphify-out/`. +Extraction produces a transient, record-only `GraphBuildData`; every durable +consumer receives a generation-safe `LoadedGraph` around Helix's native immutable +graph. Embedded, on-disk Helix is the only durable graph/index store; +NetworkX is neither imported nor installed in production. + +A build writes topology to an inactive generation, loads an immutable native +snapshot for clustering and analysis, then stages topology and all durable state +together. Counts and a SHA-256 checksum are verified before the active-generation +pointer changes. A failed or interrupted build leaves the previous generation and +its incremental hashes active. ## Module responsibilities | Module | Function | Input → Output | |--------|----------|----------------| -| `detect.py` | `collect_files(root)` | directory → `[Path]` filtered list | +| `detect.py` | `detect(root)` | directory → typed file lists and Helix-backed incremental state | | `extract.py` | `extract(path)` | file path → `{nodes, edges}` dict | -| `build.py` | `build_graph(extractions)` | list of extraction dicts → `nx.Graph` | -| `cluster.py` | `cluster(G)` | graph → graph with `community` attr on each node | +| `build.py` | `build_from_json(extraction)` | extraction dict → transient record-only `GraphBuildData` | +| `cluster.py` | `cluster(G)` | native snapshot → weighted Leiden membership | | `analyze.py` | `analyze(G)` | graph → analysis dict (god nodes, surprises, questions) | | `report.py` | `render_report(G, analysis)` | graph + analysis → GRAPH_REPORT.md string | -| `export.py` | `export(G, out_dir, ...)` | graph → Obsidian vault, graph.json, graph.html, graph.svg | +| `export.py` | `export(G, out_dir, ...)` | native snapshot → Obsidian vault, GraphML, Cypher, HTML, SVG | | `callflow_html.py` | `write_callflow_html(...)` | graphify-out files → Mermaid architecture/call-flow HTML | | `ingest.py` | `ingest(url, ...)` | URL → file saved to corpus dir | | `cache.py` | `check_semantic_cache / save_semantic_cache` | files → (cached, uncached) split | | `security.py` | validation helpers | URL / path / label → validated or raises | | `validate.py` | `validate_extraction(data)` | extraction dict → raises on schema errors | -| `serve.py` | `start_server(graph_path)` | graph file path → MCP stdio server | -| `watch.py` | `watch(root, flag_path)` | directory → writes flag file on change | -| `benchmark.py` | `run_benchmark(graph_path)` | graph file → corpus vs subgraph token comparison | +| `serve.py` | `serve(store_path)` | active Helix generation → reloadable MCP stdio server | +| `watch.py` | `watch(root)` | changes → atomic embedded Helix rebuild | +| `benchmark.py` | `run_benchmark(store_path)` | active Helix graph → corpus vs subgraph token comparison | + +## Durable generation schema + +Every active generation contains graph metadata, ordered typed nodes and edges, +communities and names, analysis/report inputs, content and semantic hashes, +extraction cache, extractor state, learning/provenance, and semantic-build +metadata. There are no production topology, cache, label, analysis, or learning +sidecars. + +Helix maintains unique equality indexes on `GraphifyNode(storage_key)` and +`GraphifyControl(control_key)`. `storage_key` is the generation plus Helix's +canonical typed external ID, so values such as `1`, `"1"`, and `true` cannot +collide. Semantic relations are native edge labels; they are not duplicated in +the edge attribute blob. + +Human-facing reports and explicit exports remain files. Existing Obsidian +application configuration remains untouched. ## Extraction output schema @@ -69,7 +95,7 @@ All external input passes through `graphify/security.py` before use: - URLs → `validate_url()` (http/https only) + `_NoFileRedirectHandler` (blocks file:// redirects) - Fetched content → `safe_fetch()` / `safe_fetch_text()` (size cap, timeout) -- Graph file paths → `validate_graph_path()` (must resolve inside `graphify-out/`) +- Helix store paths → `validate_store_path()` (must be an existing store directory) - Node labels → `sanitize_label()` (strips control chars, caps 256 chars, HTML-escapes) See `SECURITY.md` for the full threat model. @@ -82,4 +108,8 @@ One test file per module under `tests/`. Run with: pytest tests/ -q ``` -All tests are pure unit tests - no network calls, no file system side effects outside `tmp_path`. +The default suite includes unit, migration, corruption, atomicity, concurrency, +native algorithm, and embedded close/reopen tests. The native tests require the +pinned `helix-db-embedded` wheel. They are skipped in minimal source-only +environments and mandatory in CI. NetworkX parity/performance is isolated under +`benchmarks/` and is never a production dependency. diff --git a/Dockerfile b/Dockerfile index a313833c5..cb213dd73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,10 @@ # # Build: docker build -t graphify . # Run: docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ -# /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +# /data/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" # # Builds from source so the image includes the Streamable HTTP transport even -# before it lands on PyPI. The graph.json is mounted at runtime (-v), never +# before it lands on PyPI. The native Helix store is mounted at runtime (-v), never # baked into the image. FROM python:3.12-slim @@ -22,4 +22,4 @@ USER graphify EXPOSE 8080 ENTRYPOINT ["python", "-m", "graphify.serve"] -CMD ["/data/graph.json", "--transport", "http", "--host", "0.0.0.0", "--port", "8080"] +CMD ["/data/graph.helix", "--transport", "http", "--host", "0.0.0.0", "--port", "8080"] diff --git a/README.md b/README.md index 970ebf179..77ef073f6 100644 --- a/README.md +++ b/README.md @@ -48,13 +48,13 @@ Then, in your AI assistant: /graphify . ``` -That's it. You get **three files**: +That's it. You get **three outputs**: ``` graphify-out/ ├── graph.html open in any browser — click nodes, filter, search ├── GRAPH_REPORT.md the highlights: key concepts, surprising connections, suggested questions -└── graph.json the full graph — query it anytime without re-reading your files +└── graph.helix/ the embedded native graph — query it anytime without re-reading your files ``` **Works in** Claude Code, Cursor, Codex, Gemini CLI, GitHub Copilot, and 15+ more — [pick your platform](#install). @@ -101,7 +101,7 @@ What you get out of the box: | **God nodes** | The most-connected concepts, so you see what everything flows through | | **Communities** | The graph split into subsystems (Leiden), with LLM-free labels | | **Cross-file links** | `calls` / `imports` / `inherits` / `mixes_in` resolved across ~40 languages via tree-sitter AST | -| **Query, path, explain** | Ask a question, trace the path between two things, or explain one concept, all against `graph.json` | +| **Query, path, explain** | Ask a question, trace the path between two things, or explain one concept, all against the active `graph.helix` generation | | **Rationale + doc refs** | `# NOTE:` / `# WHY:` comments and ADR/RFC citations become first-class nodes linked to the code | | **Beyond code** | Docs, PDFs, images, and video/audio all map into the same graph | | **Local-first** | Code is parsed locally with tree-sitter (no LLM, nothing leaves your machine); only the semantic pass over docs/media calls a backend, and only if you configure one | @@ -134,10 +134,7 @@ Every system ran on the same harness with the same model and budgets, scored by brew install python@3.12 uv ``` -**Windows quick install:** -```powershell -winget install astral-sh.uv -``` +**Windows (PowerShell):** install Python from python.org, then run `py -m pip install graphifyy` or install `uv` and run `uv tool install graphifyy`. Native x86_64 Windows is part of the supported test matrix; installation requires the matching public `helix-db-embedded` wheel. **Ubuntu/Debian:** ```bash @@ -192,7 +189,7 @@ for example `graphify claude install --project` or `graphify codex install --pro > **Running with `uvx` / `uv tool run` instead of installing?** Name the package, not the command: `uvx --from graphifyy graphify install`. Plain `uvx graphify …` fails (`No solution found … no versions of graphify`) because `uv tool run` reads the first word as a *package*, and the package is `graphifyy` — the `graphify` command lives inside it. -> **Avoid `pip install` on Mac/Windows** if possible. The skill resolves Python at runtime from `graphify-out/.graphify_python`; if that points to a different environment than where `pip` installed the package, you'll get `ModuleNotFoundError: No module named 'graphify'`. `uv tool install` and `pipx install` isolate the package in their own env and avoid this entirely. +> **Avoid `pip install` on Mac** if possible. The skill resolves Python at runtime from `graphify-out/.graphify_python`; if that points to a different environment than where `pip` installed the package, you'll get `ModuleNotFoundError: No module named 'graphify'`. `uv tool install` and `pipx install` isolate the package in their own env and avoid this entirely. > **Git hooks and uv tool / pipx:** `graphify hook install` embeds the current interpreter path directly into the hook scripts at install time, so the post-commit hook fires correctly even in GUI git clients and CI runners where `~/.local/bin` is not on PATH. If you reinstall or upgrade graphify, re-run `graphify hook install` to refresh the embedded path. @@ -204,7 +201,7 @@ for example `graphify claude install --project` or `graphify codex install --pro | Platform | Install command | |----------|----------------| | Claude Code (Linux/Mac) | `graphify install` | -| Claude Code (Windows) | `graphify install` (auto-detected) or `graphify install --platform windows` | +| Claude Code (Windows) | `graphify install` (auto-detected) or `graphify install --platform windows`; native graph builds are supported | | CodeBuddy | `graphify install --platform codebuddy` | | Codex | `graphify install --platform codex` | | OpenCode | `graphify install --platform opencode` | @@ -368,7 +365,7 @@ You can also set `GRAPHIFY_GOOGLE_WORKSPACE=1`. Graphify exports shortcuts into /graphify . --cluster-only # rerun clustering without re-extracting /graphify . --cluster-only --resolution 1.5 # more granular communities /graphify . --cluster-only --exclude-hubs 99 # suppress utility super-hubs from god-node rankings -/graphify . --no-viz # skip the HTML, just the report + JSON +/graphify . --no-viz # skip HTML; keep the report + native store /graphify . --wiki # build a markdown wiki from the graph graphify export callflow-html # Mermaid architecture/call-flow HTML (auto-regenerates on every git commit if hook is installed) @@ -380,7 +377,7 @@ graphify export callflow-html # Mermaid architecture/call-flow HTML (auto-r /graphify add # transcribe and add a video graphify hook install # auto-rebuild on git commit -graphify merge-graphs a.json b.json # combine two graphs +graphify global add ./project-a --as project-a # register a project in the native global graph graphify prs # PR dashboard: CI state, review status, worktree mapping graphify prs 42 # deep dive on PR #42 with graph impact @@ -416,20 +413,19 @@ dist/ ## Team setup -`graphify-out/` is meant to be committed to git so everyone on the team starts with a map. +`graphify-out/graph.helix` is a local embedded store. Share source and rebuild it per checkout; do not merge native store internals across branches. **Recommended `.gitignore` additions:** ``` -graphify-out/cost.json # local only -# graphify-out/cache/ # optional: commit for speed, skip to keep repo small +graphify-out/graph.helix/ # local embedded store ``` -> `manifest.json` is now portable — keys are stored as relative paths and re-anchored on load, so committing it is safe and avoids a full rebuild on first checkout. +Hashes, extraction cache, analysis, labels, learning state, and generation metadata all live inside the Helix store. **Workflow:** -1. One person runs `/graphify .` and commits `graphify-out/`. -2. Everyone pulls — their assistant reads the graph immediately. -3. Run `graphify hook install` to auto-rebuild after each commit (AST only, no API cost). This also sets up a git merge driver so `graph.json` is never left with conflict markers — two devs committing in parallel get their graphs union-merged automatically. +1. Each developer runs `/graphify .` after checkout to create the local native store. +2. Run `graphify hook install` to update the active generation after commits. +3. Helix serializes writers and keeps readers on immutable snapshots while a new generation activates. 4. When docs or papers change, run `/graphify --update` to refresh those nodes. --- @@ -439,18 +435,18 @@ graphify-out/cost.json # local only ```bash # query the graph from the terminal graphify query "show the auth flow" -graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json +graphify query "what connects DigestAuth to Response?" --store graphify-out/graph.helix # expose the graph as an MCP server (for repeated tool-call access) -python -m graphify.serve graphify-out/graph.json -python -m graphify.serve --graph graphify-out/graph.json # --graph flag also accepted +python -m graphify.serve graphify-out/graph.helix +python -m graphify.serve --graph graphify-out/graph.helix # --graph flag also accepted # register with Kimi Code: -kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.json +kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.helix # or serve over HTTP so a whole team points at one URL (no local graphify needed): -python -m graphify.serve graphify-out/graph.json --transport http --port 8080 -python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +python -m graphify.serve graphify-out/graph.helix --transport http --port 8080 +python -m graphify.serve graphify-out/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ``` The MCP server gives your assistant structured access: `query_graph`, `get_node`, `get_neighbors`, `shortest_path`, `list_prs`, `get_pr_impact`, `triage_prs`. @@ -475,7 +471,7 @@ The default `127.0.0.1` bind is loopback-only. Set `--host 0.0.0.0` **and** `--a ```bash docker build -t graphify . docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ - /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" + /data/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ``` > **WSL / Linux note:** Ubuntu ships `python3`, not `python`. Use a venv to avoid conflicts: @@ -521,7 +517,6 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `GRAPHIFY_QUERY_LOG` | Enable the query log and write it to this path instead of the default | optional — off unless this or `_ENABLE` is set | | `GRAPHIFY_QUERY_LOG_DISABLE` | Set to `1` to force the query log off (wins over the enable vars) | optional | | `GRAPHIFY_QUERY_LOG_RESPONSES` | When the log is enabled, also record full subgraph responses (off by default) | optional | -| `GRAPHIFY_MAX_GRAPH_BYTES` | Override the 512 MiB graph.json size cap — e.g. `700MB`, `2GB`, or plain bytes | optional — useful for very large corpora | | `GRAPHIFY_LLM_TEMPERATURE` | Override LLM temperature for semantic extraction — e.g. `0.7`, or `none` to omit | optional — auto-omitted for o1/o3/o4/gpt-5 reasoning models | --- @@ -551,14 +546,14 @@ The PyPI package is `graphifyy`; `graphify` is only the command it provides. `uv **`python -m graphify` works but `graphify` command doesn't** Your shell's `PATH` doesn't include the bin directory the command was installed to. Prefer `uv tool install` / `pipx install` over plain `pip`, then run `uv tool update-shell` / `pipx ensurepath` and open a new terminal (see the install notes above). -**`/graphify .` causes "path not recognized" in PowerShell** -PowerShell treats a leading `/` as a path separator. Use `graphify .` (no slash) on Windows. +**Graphify fails to install on Windows** +Windows is supported natively—WSL and downloaded-DLL workarounds are not required or accepted. If pip reports that no compatible `helix-db-embedded` distribution exists, the matching public `win_amd64` wheel has not reached PyPI yet; use the release whose matching public wheel is available. **Graph has fewer nodes after `--update` or rebuild** If a refactor deleted files, the old nodes linger. Pass `--force` (or set `GRAPHIFY_FORCE=1`) to overwrite even when the rebuild has fewer nodes. **`extract` exits with "extraction was incomplete ... refusing to overwrite"** -When an extraction pass crashes or a walk can't fully read the corpus, the run would be smaller than a complete one, so `graphify extract` refuses to overwrite a larger existing graph with the partial result (protecting your `graph.json`). Fix the underlying failure and re-run, or pass `--allow-partial` to overwrite anyway. +When extraction fails, Graphify does not activate the incomplete generation. Fix the underlying failure and rerun; existing readers remain pinned to the prior valid generation. **Graph has duplicate nodes for the same entity (ghost duplicates)** Ghost duplicates (same symbol appearing twice — once from AST extraction with a source location, once from semantic extraction without) are now automatically merged at build time. If you see this in a graph built before v0.8.33, run a full re-extract to clean up: @@ -581,14 +576,14 @@ graphify extract . --mode deep --token-budget 4000 # smaller inpu With a cloud gateway like OpenRouter, prefer `--backend openai` (set `OPENAI_BASE_URL`) over the Ollama shim — it's a cleaner OpenAI-compatible path. If the model has its own max-output ceiling, lowering `--token-budget` is the reliable lever. **Graph HTML is too large to open in a browser (>5000 nodes)** -Skip HTML generation and use the JSON directly: +Skip HTML generation and query the native store directly: ```bash graphify cluster-only ./my-project --no-viz graphify query "..." ``` -**`graph.json` has conflict markers after two devs commit at once** -Run `graphify hook install` — it sets up a git merge driver that union-merges `graph.json` automatically so conflicts never happen. +**A native store was copied or merged between branches** +Rebuild from source. Native stores are generation directories, not mergeable source artifacts. **Extraction returns empty nodes/edges for docs or PDFs** Docs, PDFs, and images require an LLM call — code-only corpora need no key. Check that your API key is set and the backend is correct: @@ -604,10 +599,9 @@ graphify install # overwrites the skill file ``` **Claude Code prompt cache invalidated after every `graphify extract`** -Graphify writes output files (`graph.json`, `graphify-out/`) into the workspace. If those paths aren't ignored, every write invalidates Claude Code's prompt cache, forcing a full re-upload at cache-write rates on the next turn. Add them to `.claudeignore`: +Graphify writes its embedded store and exports under `graphify-out/`. If that path is not ignored, every write can invalidate Claude Code's prompt cache. Add it to `.claudeignore`: ```text # .claudeignore -graph.json graphify-out/ ``` @@ -643,13 +637,17 @@ graphify-out/ /graphify query "..." --dfs --budget 1500 /graphify path "DigestAuth" "Response" /graphify explain "SwinTransformer" +graphify impact --base origin/main # PR impact from active Helix generation +graphify update . # reuse embedded extraction records, atomically replace +graphify cluster # rerun native weighted Leiden against active Helix +graphify analyze # refresh durable analysis in one generation graphify save-result --question "Q" --answer "A" --nodes Foo Bar --outcome useful # record how a Q&A turned out (work memory; outcome ∈ useful|dead_end|corrected) graphify reflect # aggregate graphify-out/memory/ outcomes into reflections/LESSONS.md graphify reflect --if-stale # no-op when LESSONS.md is already newer than every input (cheap to run each session) graphify reflect --out docs/LESSONS.md # write the lessons doc somewhere else -graphify reflect --graph graphify-out/graph.json # group lessons by community + write the work-memory overlay (.graphify_learning.json) - # the overlay tags nodes preferred/tentative/contested (recency-weighted, with provenance); +graphify reflect --graph graphify-out/graph.helix # group lessons by community + commit work-memory state + # native state tags nodes preferred/tentative/contested (recency-weighted, with provenance); # graphify explain / query then show a "Lesson:" hint, flagged "code changed — re-verify" when the source moved on graphify uninstall # remove from all platforms in one shot @@ -722,7 +720,7 @@ graphify extract ./src --no-gitignore # include git-ignored source; sti graphify extract ./docs --mode deep # richer semantic extraction via extended system prompt graphify extract ./docs --no-cluster # raw extraction only, skip clustering graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) -graphify extract ./docs --force # overwrite graph.json even if new graph has fewer nodes (use after refactors or to clear ghost duplicates) +graphify extract ./docs --force # force a full source rescan and activate a new native generation graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) graphify extract ./docs --global --as myrepo # extract and register into the cross-project global graph GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # raise output cap for dense corpora @@ -732,7 +730,7 @@ graphify export callflow-html --max-sections 8 # cap generated architecture graphify export callflow-html --output docs/arch.html graphify export callflow-html ./some-repo/graphify-out -graphify global add graphify-out/graph.json --as myrepo # register a project graph into ~/.graphify/global-graph.json +graphify global add graphify-out/graph.helix --as myrepo # register a project store into ~/.graphify/global-graph.helix graphify global remove myrepo # remove a project from the global graph graphify global list # show all registered repos + node/edge counts graphify global path # print path to the global graph file @@ -747,7 +745,6 @@ graphify prs --repo owner/repo # run against a different GitHub repo GRAPHIFY_TRIAGE_BACKEND=kimi graphify prs --triage # use a specific backend for triage graphify clone https://github.com/karpathy/nanoGPT -graphify merge-graphs a.json b.json --out merged.json graphify --version # print installed version graphify watch ./src graphify check-update ./src @@ -755,7 +752,7 @@ graphify update ./src graphify update ./src --no-cluster # skip reclustering, write raw AST graph only graphify update ./src --force # overwrite even if new graph has fewer nodes graphify cluster-only ./my-project -graphify cluster-only ./my-project --graph path/to/graph.json # custom graph location +graphify cluster-only ./my-project --store path/to/graph.helix # custom Helix store directory graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # parallel community labeling (large graphs) graphify cluster-only ./my-project --resolution 1.5 # more, smaller communities graphify cluster-only ./my-project --exclude-hubs 99 # exclude p99 degree nodes from partitioning diff --git a/SECURITY.md b/SECURITY.md index 795454b21..b962c6448 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -38,7 +38,11 @@ graphify is a **local development tool**. It runs as a Claude Code skill and opt | YAML frontmatter injection | `_yaml_str()` escapes backslashes, double quotes, and newlines before embedding user-controlled strings (webpage titles, query questions) in YAML frontmatter. | | Encoding crashes on source files | All tree-sitter byte slices decoded with `errors="replace"` - non-UTF-8 source files degrade gracefully instead of crashing extraction. | | Symlink traversal | `os.walk(..., followlinks=False)` is explicit throughout `detect.py`. | -| Corrupted graph.json | `_load_graph()` in `serve.py` wraps `json.JSONDecodeError` and prints a clear recovery message instead of crashing. | +| Corrupted or mixed-generation store | Active Helix generations carry counts and SHA-256 checksums; readers reject mismatches before exposing the graph. | +| Partial/interrupted build | Topology and durable state remain inactive until complete verification and one active-generation pointer update. | +| Competing embedded writers | A process lock permits one writer; read-only handles reject every write path. | +| Public or mismatched Helix build | Startup validates the exact pinned SDK revision, version, and native graph API payload and fails closed. | +| MCP HTTP exposure | Streamable HTTP binds only to loopback addresses; non-loopback hosts are rejected. Use a trusted authenticated proxy if remote access is required. | ### What graphify does NOT do @@ -46,6 +50,7 @@ graphify is a **local development tool**. It runs as a Claude Code skill and opt - Does not execute code from source files (tree-sitter parses ASTs - no eval/exec) - Does not use `shell=True` in any subprocess call - Does not store credentials or API keys +- Does not persist Graphify graph/index state in JSON sidecars ### Optional network calls diff --git a/benchmarks/QUALIFICATION.md b/benchmarks/QUALIFICATION.md new file mode 100644 index 000000000..9e7f88ff6 --- /dev/null +++ b/benchmarks/QUALIFICATION.md @@ -0,0 +1,93 @@ +# Release qualification: public Helix `0.2.0b3` (draft) + +Graphify is pinned to the matching public PyPI releases +`helix-db==0.2.0b3` and `helix-db-embedded==0.2.0b3`. It uses ordinary public +`helixdb` imports and rejects incompatible SDK versions, missing embedded +payloads, malformed store paths, and unavailable required public APIs. No +Helix source checkout, Git dependency, private wheel, locally compiled +runtime, direct UniFFI import, dynamic native loader, or SDK monkeypatch is +used. + +Native Windows x86_64 is a supported Graphify target. Qualification still +requires a normal public `win_amd64` embedded wheel for the exact pinned +version. Public b3 currently publishes macOS universal2 and Linux +x86_64/aarch64 artifacts, but no Windows artifact, so the PR remains draft. + +## Graphify acceptance + +- Complete local suite: **2,925 passed, 3 skipped**. The Python 3.12 clean + environment accounted for the same behavior surface after installing the + declared development group, including wheel-packaging tests. +- Complete Python 3.10 suite: **2,912 passed, 16 skipped**. The additional + skips are the expected Python-version-gated optional video coverage; no + Helix or retained core behavior was skipped. +- CI matrix: Linux x86_64 on Python 3.10 and 3.12, Linux aarch64 on Python + 3.12, macOS universal2 on Python 3.12, and native Windows x86_64 on Python + 3.10 and 3.12. +- All four graph kinds, typed IDs, keyed multiedges, self-loops, hyperedges, + atomic activation, retained rollback, corruption, writer exclusion, + concurrent readers, generation hot reload, and read-only enforcement: passed. +- Incremental update and deletion, extraction cache, analysis, learning state, + clustering, affected/PR analysis, global aggregation, hooks, watch, exports, + MCP stdio, and MCP Streamable HTTP: passed. +- Generated skill check (134 artifacts), host coverage audit, schema singleton, + monolith contract, and always-on contract: passed. +- Ruff, scoped Pyright for the native integration surface, Bandit medium/high, + locked-runtime pip-audit, source compilation, package build, and + `git diff --check`: passed locally. +- Wheel installation in a clean Python 3.12 environment: passed. The installed + production dependency set contains neither NetworkX nor graspologic. +- Installed-wheel CLI flow: build native store, query, and shortest path: passed. + +Production source contains no NetworkX/graspologic imports or compatibility +graph. Legacy JSON graph files are never migrated, read, modified, or deleted; +the watch path emits one obsolete-format warning and requires a source rebuild. +Obsidian's `.obsidian/graph.json` remains solely because that filename belongs +to Obsidian's presentation configuration, not Graphify storage. + +## Benchmark qualification + +The isolated benchmark comparator is the only environment that installs +NetworkX. The deterministic parity corpus and full raw measurements are in +[`parity-networkx.json`](parity-networkx.json) and +[`helix-vs-networkx-2026-07-19.json`](helix-vs-networkx-2026-07-19.json); reproduction commands and +interpreted results are in [`RESULTS.md`](RESULTS.md). + +The public b3 candidate **does not pass the release gates**: + +| Graph | Helix ingest | Helix cold open | Peak RSS (informational) | Active store | Slowest absolute hot op | +|---|---:|---:|---:|---:|---:| +| 5k / 15k | 8.17s | 1.07s | 232.3 MiB | 35.12 MiB | 20.57ms | +| 20k / 60k | 32.55s | 4.88s | 647.1 MiB | 143.45 MiB | 8.93ms | + +At 20k/60k, weighted Leiden, sampled node centrality, and sampled edge +centrality are respectively **9.76x**, **9.93x**, and **4.13x** faster than the +current v8 NetworkX/JSON comparator, so the three native analytics gates pass. +Absolute hot-operation limits also pass. Build, 1% update, cold-open, active +and post-update storage, and relative warm-operation gates fail. + +Default retention deletes the inactive generation through public Helix +operations, but b3 does not physically reclaim enough embedded-store space; +the 20k/60k store grows from 143.45 MiB to 298.80 MiB after the 1% update. +Graphify deliberately does not manipulate SST/WAL files or call private +maintenance APIs. Peak RSS is reported but is non-blocking, as required. + +The public ERPNext corpus was independently reproduced at commit +`ea5c648ab04a2b30c5c238f6cb299c4237ff1c1e`. Helix now matches the loaded v8 +topology exactly at 25,443 nodes and 59,142 edges, but fresh build is 99.15s +versus 41.30s, cold open is 7.95s versus 0.33s, active storage is 10.34x v8, +and a representative steady query is 3.52s versus 0.05s. Raw measurements are +in [`report-erpnext-2026-07-19.json`](report-erpnext-2026-07-19.json). + +The report's other four corpora, exact gold queries, harness scripts, and raw +results were not distributed or attached. The five-corpus gold-recall table +therefore remains outstanding; the public ERPNext and synthetic results are +not a substitute for those missing inputs. + +## Conclusion + +The integration is correct enough for a draft PR, but is not production-ready. +It must remain draft until the exact public package pair provides a passing +Windows wheel, all CI platforms pass without core skips, the engineers' real +corpora and gold queries pass, and the failed public-package performance and +disk-reclamation gates are resolved without hidden workarounds. diff --git a/benchmarks/RESULTS.md b/benchmarks/RESULTS.md new file mode 100644 index 000000000..bd57a3c23 --- /dev/null +++ b/benchmarks/RESULTS.md @@ -0,0 +1,74 @@ +# Helix vs NetworkX benchmark + +Run on Python 3.12.13 and macOS arm64 with the matching public +`helix-db==0.2.0b3` / `helix-db-embedded==0.2.0b3` pair and NetworkX 3.6.1. +NetworkX was installed only in the isolated benchmark environment. Both +backends use the same deterministic topology and sampled betweenness uses 100 +sources with seed 42. Peak RSS is reported but is not a release gate. + +Lower is better. Exact samples and every acceptance check are retained in +[`helix-vs-networkx-2026-07-19.json`](helix-vs-networkx-2026-07-19.json). + +| Graph | Backend | Ingest | 1% update | Cold reopen | Hot open | 20 neighbors | BFS d=4 | 5 paths | +|---|---|---:|---:|---:|---:|---:|---:|---:| +| 5k / 15k | NetworkX | 0.076s | 0.045s | 0.026s | n/a | 0.004ms | 1.39ms | 0.11ms | +| 5k / 15k | Helix | 8.167s | 10.730s | 1.073s | 5.48ms | 1.15ms | 20.57ms | 1.11ms | +| 20k / 60k | NetworkX | 0.304s | 0.192s | 0.124s | n/a | 0.003ms | 0.93ms | 0.11ms | +| 20k / 60k | Helix | 32.554s | 49.525s | 4.875s | 7.15ms | 1.10ms | 8.93ms | 3.25ms | + +| Graph | Backend | Community | Node BTW | Edge BTW | GraphML export | Peak ingest RSS | +|---|---|---:|---:|---:|---:|---:| +| 5k / 15k | NetworkX | 0.996s Louvain | 0.665s | 0.826s | 0.512s | 24.3 MiB | +| 5k / 15k | Helix | 0.095s Leiden | 0.093s | 0.301s | 0.582s | 232.3 MiB | +| 20k / 60k | NetworkX | 9.915s Louvain | 4.096s | 5.731s | 0.539s | 97.4 MiB | +| 20k / 60k | Helix | 1.015s Leiden | 0.412s | 1.386s | 2.553s | 647.1 MiB | + +| Graph | NetworkX JSON | Active Helix store | Helix after default-retention update | Eight concurrent cold reopens | +|---|---:|---:|---:|---:| +| 5k / 15k | 1.37 MiB | 35.12 MiB | 73.24 MiB | 4.03s | +| 20k / 60k | 5.58 MiB | 143.45 MiB | 298.80 MiB | 16.80s | + +## Gate result + +This candidate **does not pass the release gates** and the PR must remain +draft. Absolute hot-operation limits pass, and at 20k/60k native weighted +Leiden, sampled node centrality, and sampled edge centrality are respectively +9.76x, 9.93x, and 4.13x faster than the NetworkX comparator. Build, 1% update, +cold-open, storage, and relative warm-operation gates fail. + +Public b3 substantially improves the earlier public b1 engine result, but it +still does not reach the required 2x build/update or 3s cold-open limits. +Increasing Graphify's public write-batch size from 1,000 to 5,000 records did +not improve the remaining ingest time. + +Default retention logically deletes the inactive generation through public +Helix operations, but the embedded store does not reclaim its physical space; +the post-update store more than doubles. Graphify does not compact SST/WAL +files or call private maintenance APIs, so disk reclamation remains a public +package limitation. + +The behavioral parity result from the current candidate +passes directed path, BFS, DFS, exact node and edge betweenness, Louvain on a +golden graph, layout, relabeling, subgraph, and conversion checks. + +## Public ERPNext report reproduction + +The report-style public ERPNext run now has exact loaded-topology parity at +25,443 nodes and 59,142 edges. It still fails release qualification: fresh +build is 99.15s versus 41.30s, median cold open is 7.95s versus 0.33s, active +storage is 10.34x v8, and a representative steady query is 3.52s versus 0.05s. +The immediate unchanged-topology update passes at 46.65s versus 40.13s. See +[`report-erpnext-2026-07-19.json`](report-erpnext-2026-07-19.json) for commands, +versions, samples, and unavailable report inputs. + +Reproduce without Homebrew: + +```bash +python3.12 -m venv /tmp/graphify-networkx-benchmark +/tmp/graphify-networkx-benchmark/bin/python -m pip install -e . \ + -r benchmarks/requirements-networkx.txt +PYTHONPATH=. /tmp/graphify-networkx-benchmark/bin/python benchmarks/parity_networkx.py +PYTHONPATH=. /tmp/graphify-networkx-benchmark/bin/python \ + benchmarks/helix_vs_networkx.py \ + --out benchmarks/helix-vs-networkx-2026-07-19.json --check-gates +``` diff --git a/benchmarks/helix-vs-networkx-2026-07-19.json b/benchmarks/helix-vs-networkx-2026-07-19.json new file mode 100644 index 000000000..360cbc9c5 --- /dev/null +++ b/benchmarks/helix-vs-networkx-2026-07-19.json @@ -0,0 +1,460 @@ +{ + "helix_revision": "0.2.0b3", + "python": "3.12.13", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "pid": 55199, + "results": [ + { + "nodes": 5000, + "edges": 15000, + "networkx": { + "ingest_seconds": 0.07641220799996518, + "ingest_runs": 1, + "reopen_seconds": 0.025735499999427702, + "reopen_samples_seconds": [ + 0.025735499999427702, + 0.024884999998903368, + 0.03188929199677659 + ], + "neighbor_20_queries_seconds": 3.5420016502030194e-06, + "neighbor_samples_seconds": [ + 2.454200148349628e-05, + 4.417001036927104e-06, + 3.5420016502030194e-06, + 3.3750038710422814e-06, + 3.291999746579677e-06 + ], + "bfs_seconds": 0.001392665995808784, + "bfs_samples_seconds": [ + 0.0024065000034170225, + 0.001651625003432855, + 0.0013815839993185364, + 0.001379041001200676, + 0.001392665995808784 + ], + "shortest_path_5_queries_seconds": 0.00010879200272029266, + "shortest_path_samples_seconds": [ + 0.0003638340058387257, + 0.00010900000052060932, + 0.00010879200272029266, + 0.00010595899948384613, + 0.00010445800580782816 + ], + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": 0.9960345830040751, + "community_samples_seconds": [ + 0.9838285829973756, + 1.0004757499991683, + 0.9960345830040751 + ], + "node_betweenness_seconds": 0.665316584003449, + "node_betweenness_samples_seconds": [ + 0.665316584003449 + ], + "edge_betweenness_seconds": 0.8258883750022505, + "edge_betweenness_samples_seconds": [ + 0.8258883750022505 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 0.0450237080003717, + "graphml_export_seconds": 0.5118335419974755, + "peak_rss_delta_bytes": 25477120, + "disk_bytes": 1436547 + }, + "helix": { + "ingest_seconds": 8.167331750002631, + "ingest_runs": 1, + "reopen_seconds": 1.0732905830009258, + "reopen_samples_seconds": [ + 1.0922817909959122, + 1.0656928750031511, + 1.0732905830009258 + ], + "hot_open_seconds": 0.005478249993757345, + "hot_open_samples_seconds": [ + 0.005679333000443876, + 0.005313916997693013, + 0.005478249993757345, + 0.0054750839990447275, + 0.005643750002491288 + ], + "neighbor_20_queries_seconds": 0.0011476250001578592, + "neighbor_samples_seconds": [ + 0.0012960420062881894, + 0.0011476250001578592, + 0.0011508749958011322, + 0.0011415420012781397, + 0.0011369999992894009 + ], + "bfs_seconds": 0.020567291998304427, + "bfs_samples_seconds": [ + 0.020894415996735916, + 0.020567291998304427, + 0.020452249998925254, + 0.020468916998652276, + 0.020609833001799416 + ], + "shortest_path_5_queries_seconds": 0.0011083749996032566, + "shortest_path_samples_seconds": [ + 0.0013168330042390153, + 0.001116250001359731, + 0.0011083749996032566, + 0.0010899580011027865, + 0.0010933750018011779 + ], + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": 0.0947054170028423, + "community_samples_seconds": [ + 0.09471074999601115, + 0.0947054170028423, + 0.09400404199550394 + ], + "node_betweenness_seconds": 0.09330433300056029, + "node_betweenness_samples_seconds": [ + 0.09330433300056029 + ], + "edge_betweenness_seconds": 0.3006899579995661, + "edge_betweenness_samples_seconds": [ + 0.3006899579995661 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 10.729899457997817, + "graphml_export_seconds": 0.5823157080012606, + "eight_concurrent_reopens_seconds": 4.033307792000414, + "peak_rss_delta_bytes": 243630080, + "disk_after_ingest_bytes": 36827950, + "disk_after_update_bytes": 76794602 + } + }, + { + "nodes": 20000, + "edges": 60000, + "networkx": { + "ingest_seconds": 0.3044721250043949, + "ingest_runs": 1, + "reopen_seconds": 0.12366070800635498, + "reopen_samples_seconds": [ + 0.12366070800635498, + 0.10231733399996301, + 0.13137324999843258 + ], + "neighbor_20_queries_seconds": 3.4579934435896575e-06, + "neighbor_samples_seconds": [ + 2.2000000171829015e-05, + 4.082998202648014e-06, + 3.4579934435896575e-06, + 3.25000291923061e-06, + 3.416003892198205e-06 + ], + "bfs_seconds": 0.0009334999995189719, + "bfs_samples_seconds": [ + 0.0011756669991882518, + 0.0010096670011989772, + 0.0007989160003489815, + 0.0007129590012482367, + 0.0009334999995189719 + ], + "shortest_path_5_queries_seconds": 0.0001099579967558384, + "shortest_path_samples_seconds": [ + 0.00017779199697542936, + 0.00011137499677715823, + 0.0001099579967558384, + 0.00010837500303750858, + 0.00010750000365078449 + ], + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": 9.914627540994843, + "community_samples_seconds": [ + 9.753872833003697, + 9.914627540994843, + 10.012469834000512 + ], + "node_betweenness_seconds": 4.096053250003024, + "node_betweenness_samples_seconds": [ + 4.096053250003024 + ], + "edge_betweenness_seconds": 5.731195250002202, + "edge_betweenness_samples_seconds": [ + 5.731195250002202 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 0.1919678330013994, + "graphml_export_seconds": 0.5387877500033937, + "peak_rss_delta_bytes": 102105088, + "disk_bytes": 5852869 + }, + "helix": { + "ingest_seconds": 32.553641709004296, + "ingest_runs": 1, + "reopen_seconds": 4.8753892079985235, + "reopen_samples_seconds": [ + 4.974821375006286, + 4.8753892079985235, + 4.873887665999064 + ], + "hot_open_seconds": 0.007152040998334996, + "hot_open_samples_seconds": [ + 0.007963042000483256, + 0.0071519999983138405, + 0.006855957995867357, + 0.007557749995612539, + 0.007152040998334996 + ], + "neighbor_20_queries_seconds": 0.0011049160020775162, + "neighbor_samples_seconds": [ + 0.0012251250009285286, + 0.001101458001357969, + 0.0011022909966413863, + 0.0011049160020775162, + 0.0011372919980203733 + ], + "bfs_seconds": 0.008930916999815963, + "bfs_samples_seconds": [ + 0.009267082998121623, + 0.008957958001701627, + 0.008930916999815963, + 0.008895042003132403, + 0.008882750000339001 + ], + "shortest_path_5_queries_seconds": 0.0032470840014866553, + "shortest_path_samples_seconds": [ + 0.004328625000198372, + 0.00329987499571871, + 0.0032433340020361356, + 0.0032470840014866553, + 0.003222624996851664 + ], + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": 1.0154825419958797, + "community_samples_seconds": [ + 1.0154825419958797, + 1.0692213329966762, + 1.0087809999968158 + ], + "node_betweenness_seconds": 0.41240491600183304, + "node_betweenness_samples_seconds": [ + 0.41240491600183304 + ], + "edge_betweenness_seconds": 1.3862965000007534, + "edge_betweenness_samples_seconds": [ + 1.3862965000007534 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 49.52538345799985, + "graphml_export_seconds": 2.5530614579984103, + "eight_concurrent_reopens_seconds": 16.802997667000454, + "peak_rss_delta_bytes": 678559744, + "disk_after_ingest_bytes": 150420706, + "disk_after_update_bytes": 313312766 + } + } + ], + "acceptance": { + "passed": false, + "checks": [ + { + "name": "5000/15000 ingest vs v8", + "actual": 106.88516879405387, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 1% update vs v8", + "actual": 238.31665437038717, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 cold open seconds", + "actual": 1.0732905830009258, + "limit": 0.5, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 cold open vs v8", + "actual": 41.704671874445545, + "limit": 5.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 active store vs v8", + "actual": 25.63643932290416, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 post-update store vs v8", + "actual": 53.4577720046751, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 hot_open_seconds", + "actual": 0.005478249993757345, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 neighbor_20_queries_seconds", + "actual": 0.0011476250001578592, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 neighbor_20_queries_seconds vs v8", + "actual": 324.00464864043187, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 bfs_seconds", + "actual": 0.020567291998304427, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 bfs_seconds vs v8", + "actual": 14.768287629770176, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 shortest_path_5_queries_seconds", + "actual": 0.0011083749996032566, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 shortest_path_5_queries_seconds vs v8", + "actual": 10.188019081263908, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 ingest vs v8", + "actual": 106.91829903487684, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 1% update vs v8", + "actual": 257.98792789226735, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 cold open seconds", + "actual": 4.8753892079985235, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 cold open vs v8", + "actual": 39.42553205944749, + "limit": 5.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 active store vs v8", + "actual": 25.70033704837747, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 post-update store vs v8", + "actual": 53.53148447368291, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 clustering speedup", + "actual": 9.763464295021894, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 node centrality speedup", + "actual": 9.93211547940136, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 edge centrality speedup", + "actual": 4.134177104247963, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 hot_open_seconds", + "actual": 0.007152040998334996, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 neighbor_20_queries_seconds", + "actual": 0.0011049160020775162, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 neighbor_20_queries_seconds vs v8", + "actual": 319.5251871069282, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 bfs_seconds", + "actual": 0.008930916999815963, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 bfs_seconds vs v8", + "actual": 9.567131231299435, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 shortest_path_5_queries_seconds", + "actual": 0.0032470840014866553, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 shortest_path_5_queries_seconds vs v8", + "actual": 29.530221514464305, + "limit": 2.0, + "comparison": "<=", + "passed": false + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/helix-vs-networkx.json b/benchmarks/helix-vs-networkx.json new file mode 100644 index 000000000..4018784e6 --- /dev/null +++ b/benchmarks/helix-vs-networkx.json @@ -0,0 +1,460 @@ +{ + "helix_revision": "0.2.0b3", + "python": "3.12.13", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "pid": 41164, + "results": [ + { + "nodes": 5000, + "edges": 15000, + "networkx": { + "ingest_seconds": 0.07608495800377568, + "ingest_runs": 1, + "reopen_seconds": 0.027275708001980092, + "reopen_samples_seconds": [ + 0.02667249999649357, + 0.027275708001980092, + 0.03735450000385754 + ], + "neighbor_20_queries_seconds": 5.041001713834703e-06, + "neighbor_samples_seconds": [ + 2.2290994820650667e-05, + 8.75000114319846e-06, + 5.041001713834703e-06, + 4.624998837243766e-06, + 4.542001988738775e-06 + ], + "bfs_seconds": 0.001965709001524374, + "bfs_samples_seconds": [ + 0.003230207999877166, + 0.0022959170019021258, + 0.0019293750010547228, + 0.001913125001010485, + 0.001965709001524374 + ], + "shortest_path_5_queries_seconds": 0.0001400420005666092, + "shortest_path_samples_seconds": [ + 0.00046541699703084305, + 0.0001400420005666092, + 0.00014104200090514496, + 0.00013374999980442226, + 0.00013350000517675653 + ], + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": 1.300195583004097, + "community_samples_seconds": [ + 1.2948922499999753, + 1.3108603340006084, + 1.300195583004097 + ], + "node_betweenness_seconds": 0.8536021670006448, + "node_betweenness_samples_seconds": [ + 0.8536021670006448 + ], + "edge_betweenness_seconds": 1.0998514579987386, + "edge_betweenness_samples_seconds": [ + 1.0998514579987386 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 0.058347334001155104, + "graphml_export_seconds": 0.22606754099979298, + "peak_rss_delta_bytes": 25526272, + "disk_bytes": 1436547 + }, + "helix": { + "ingest_seconds": 10.135198459000094, + "ingest_runs": 1, + "reopen_seconds": 1.4570273749995977, + "reopen_samples_seconds": [ + 1.4632811670016963, + 1.4570273749995977, + 1.414785832996131 + ], + "hot_open_seconds": 0.007826082997780759, + "hot_open_samples_seconds": [ + 0.007399832997180056, + 0.00796341599925654, + 0.007826082997780759, + 0.007802540996635798, + 0.007980541995493695 + ], + "neighbor_20_queries_seconds": 0.0016158340004039928, + "neighbor_samples_seconds": [ + 0.001879417002783157, + 0.0016267910032183863, + 0.0016158340004039928, + 0.0016101250002975576, + 0.0016047089957282878 + ], + "bfs_seconds": 0.027395249999244697, + "bfs_samples_seconds": [ + 0.027870541998709086, + 0.026020999997854233, + 0.02741887500451412, + 0.027395249999244697, + 0.026016083000286017 + ], + "shortest_path_5_queries_seconds": 0.001446291003958322, + "shortest_path_samples_seconds": [ + 0.001650833000894636, + 0.0014340410052682273, + 0.0014507079977192916, + 0.0014111669952399097, + 0.001446291003958322 + ], + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": 0.12093970800196985, + "community_samples_seconds": [ + 0.12145224999403581, + 0.12093970800196985, + 0.12019025000336114 + ], + "node_betweenness_seconds": 0.12077770799805876, + "node_betweenness_samples_seconds": [ + 0.12077770799805876 + ], + "edge_betweenness_seconds": 0.394192749998183, + "edge_betweenness_samples_seconds": [ + 0.394192749998183 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 13.60693266599992, + "graphml_export_seconds": 0.7467261250058073, + "eight_concurrent_reopens_seconds": 7.142115583999839, + "peak_rss_delta_bytes": 359956480, + "disk_after_ingest_bytes": 37063727, + "disk_after_update_bytes": 77265897 + } + }, + { + "nodes": 20000, + "edges": 60000, + "networkx": { + "ingest_seconds": 0.37801691699860385, + "ingest_runs": 1, + "reopen_seconds": 0.1524028330022702, + "reopen_samples_seconds": [ + 0.1524028330022702, + 0.12498162500560284, + 0.15622158299811417 + ], + "neighbor_20_queries_seconds": 4.749999789055437e-06, + "neighbor_samples_seconds": [ + 2.7707996196113527e-05, + 5.875001079402864e-06, + 4.749999789055437e-06, + 4.499997885432094e-06, + 4.41699376096949e-06 + ], + "bfs_seconds": 0.0011056670045945793, + "bfs_samples_seconds": [ + 0.001745207999192644, + 0.001052082996466197, + 0.0011195409970241599, + 0.0010090420037158765, + 0.0011056670045945793 + ], + "shortest_path_5_queries_seconds": 0.0001388750024489127, + "shortest_path_samples_seconds": [ + 0.00020545800362015143, + 0.00014370799908647314, + 0.0001388750024489127, + 0.00013629200111608952, + 0.0001368750017718412 + ], + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": 13.141373332997318, + "community_samples_seconds": [ + 13.224506249993283, + 13.122070040997642, + 13.141373332997318 + ], + "node_betweenness_seconds": 4.9877693750022445, + "node_betweenness_samples_seconds": [ + 4.9877693750022445 + ], + "edge_betweenness_seconds": 6.920005957996182, + "edge_betweenness_samples_seconds": [ + 6.920005957996182 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 0.253822416998446, + "graphml_export_seconds": 0.6834231249958975, + "peak_rss_delta_bytes": 102203392, + "disk_bytes": 5852869 + }, + "helix": { + "ingest_seconds": 40.671021500005736, + "ingest_runs": 1, + "reopen_seconds": 6.454431124999246, + "reopen_samples_seconds": [ + 6.424296000004688, + 6.456437791995995, + 6.454431124999246 + ], + "hot_open_seconds": 0.01001570800144691, + "hot_open_samples_seconds": [ + 0.009779542000615038, + 0.009466625000641216, + 0.01001570800144691, + 0.010156750002352055, + 0.010143042003619485 + ], + "neighbor_20_queries_seconds": 0.0016046660020947456, + "neighbor_samples_seconds": [ + 0.0018574580026324838, + 0.0016118330022436567, + 0.0016046660020947456, + 0.0016024159995140508, + 0.0015940000012051314 + ], + "bfs_seconds": 0.012172125003417023, + "bfs_samples_seconds": [ + 0.013258124999993015, + 0.012863666997873224, + 0.012022124996292405, + 0.012172125003417023, + 0.012125667002692353 + ], + "shortest_path_5_queries_seconds": 0.004266167001333088, + "shortest_path_samples_seconds": [ + 0.005248583001957741, + 0.004290082993975375, + 0.004266167001333088, + 0.004225707998557482, + 0.004209208003885578 + ], + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": 1.413519665999047, + "community_samples_seconds": [ + 1.413519665999047, + 1.421890292003809, + 1.3064116250025108 + ], + "node_betweenness_seconds": 0.6120458329969551, + "node_betweenness_samples_seconds": [ + 0.6120458329969551 + ], + "edge_betweenness_seconds": 1.8529502079982194, + "edge_betweenness_samples_seconds": [ + 1.8529502079982194 + ], + "betweenness_mode": "sampled (100 sources, seed 42)", + "incremental_1pct_seconds": 63.57189174999803, + "graphml_export_seconds": 3.2455323329995736, + "eight_concurrent_reopens_seconds": 31.242825875000563, + "peak_rss_delta_bytes": 1041793024, + "disk_after_ingest_bytes": 150693438, + "disk_after_update_bytes": 313858260 + } + } + ], + "acceptance": { + "passed": false, + "checks": [ + { + "name": "5000/15000 ingest vs v8", + "actual": 133.20896435925138, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 1% update vs v8", + "actual": 233.20573080049456, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 cold open seconds", + "actual": 1.4570273749995977, + "limit": 0.5, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 cold open vs v8", + "actual": 53.4184987936454, + "limit": 5.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 active store vs v8", + "actual": 25.80056691497041, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 post-update store vs v8", + "actual": 53.785846895367854, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 hot_open_seconds", + "actual": 0.007826082997780759, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 neighbor_20_queries_seconds", + "actual": 0.0016158340004039928, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 neighbor_20_queries_seconds vs v8", + "actual": 320.5382763448465, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 bfs_seconds", + "actual": 0.027395249999244697, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 bfs_seconds vs v8", + "actual": 13.936574527562394, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "5000/15000 shortest_path_5_queries_seconds", + "actual": 0.001446291003958322, + "limit": 0.1, + "comparison": "<=", + "passed": true + }, + { + "name": "5000/15000 shortest_path_5_queries_seconds vs v8", + "actual": 10.327551720959685, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 ingest vs v8", + "actual": 107.59047987303687, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 1% update vs v8", + "actual": 250.45814511484716, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 cold open seconds", + "actual": 6.454431124999246, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 cold open vs v8", + "actual": 42.35112299325237, + "limit": 5.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 active store vs v8", + "actual": 25.746935050143783, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 post-update store vs v8", + "actual": 53.62468560290688, + "limit": 3.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 clustering speedup", + "actual": 9.296915811715476, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 node centrality speedup", + "actual": 8.149339650886992, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 edge centrality speedup", + "actual": 3.7345881870576587, + "limit": 3.0, + "comparison": ">=", + "passed": true + }, + { + "name": "20000/60000 hot_open_seconds", + "actual": 0.01001570800144691, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 neighbor_20_queries_seconds", + "actual": 0.0016046660020947456, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 neighbor_20_queries_seconds vs v8", + "actual": 337.824436496205, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 bfs_seconds", + "actual": 0.012172125003417023, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 bfs_seconds vs v8", + "actual": 11.008852532304912, + "limit": 2.0, + "comparison": "<=", + "passed": false + }, + { + "name": "20000/60000 shortest_path_5_queries_seconds", + "actual": 0.004266167001333088, + "limit": 0.5, + "comparison": "<=", + "passed": true + }, + { + "name": "20000/60000 shortest_path_5_queries_seconds vs v8", + "actual": 30.719473815328733, + "limit": 2.0, + "comparison": "<=", + "passed": false + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/helix_vs_networkx.py b/benchmarks/helix_vs_networkx.py new file mode 100644 index 000000000..2dd34f244 --- /dev/null +++ b/benchmarks/helix_vs_networkx.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +"""Reproducible Helix comparison against the current v8 NetworkX/JSON baseline.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import platform +import random +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Callable + +try: + import resource +except ImportError: # Native Windows has no resource module; RSS is informational. + resource = None + +from graphify.helix.model import EdgeData, GraphBuildData, NodeData +from graphify.helix.persistence import HelixEmbeddedStore, HelixGraphReader +from graphify.helix.state import new_state + + +SIZES = ((5_000, 15_000), (20_000, 60_000)) + + +def _peak_rss_bytes() -> int: + if resource is None: + return 0 + value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return value if platform.system() == "Darwin" else value * 1024 + + +def measure(call: Callable[[], Any]) -> tuple[Any, float, int]: + before = _peak_rss_bytes() + started = time.perf_counter() + result = call() + elapsed = time.perf_counter() - started + after = _peak_rss_bytes() + return result, elapsed, max(0, after - before) + + +def median_measure(call: Callable[[], Any], runs: int) -> tuple[Any, float, list[float]]: + samples: list[float] = [] + result: Any = None + for _ in range(runs): + result, elapsed, _ = measure(call) + samples.append(elapsed) + return result, statistics.median(samples), samples + + +def isolated_ingest_memory(backend: str, nodes: int, edges: int) -> int: + """Measure each backend in a fresh process so peak RSS cannot leak across runs.""" + output = subprocess.check_output( + [ + sys.executable, + str(Path(__file__).resolve()), + "--memory-backend", + backend, + "--memory-nodes", + str(nodes), + "--memory-edges", + str(edges), + ], + text=True, + env={**os.environ, "PYTHONPATH": os.environ.get("PYTHONPATH", ".")}, + ) + return int(output.strip()) + + +def memory_only(backend: str, nodes: int, edges: int) -> int: + root = Path(tempfile.mkdtemp(prefix="graphify-memory-benchmark-")) + try: + if backend == "networkx": + import networkx as networkx + + before = _peak_rss_bytes() + graph = topology(networkx.Graph, nodes, edges, 42) + payload = networkx.node_link_data(graph, edges="links") + (root / "graph.json").write_text( + json.dumps(payload, indent=2), encoding="utf-8" + ) + return max(0, _peak_rss_bytes() - before) + before = _peak_rss_bytes() + graph = helix_topology(nodes, edges, 42) + with HelixEmbeddedStore(root / "graph.helix") as store: + store.save_generation(graph, new_state(build={"benchmark": True})) + return max(0, _peak_rss_bytes() - before) + finally: + shutil.rmtree(root, ignore_errors=True) + + +def topology(graph_type: type, nodes: int, edges: int, seed: int): + graph = graph_type() + graph.add_nodes_from((index, {"label": f"node-{index}"}) for index in range(nodes)) + rng = random.Random(seed) + seen: set[tuple[int, int]] = set() + while len(seen) < edges: + source, target = rng.randrange(nodes), rng.randrange(nodes) + pair = (min(source, target), max(source, target)) + if source != target and pair not in seen: + seen.add(pair) + graph.add_edge(source, target, weight=1.0) + return graph + + +def helix_topology(nodes: int, edges: int, seed: int) -> GraphBuildData: + rng = random.Random(seed) + seen: set[tuple[int, int]] = set() + while len(seen) < edges: + source, target = rng.randrange(nodes), rng.randrange(nodes) + pair = (min(source, target), max(source, target)) + if source != target: + seen.add(pair) + return GraphBuildData( + nodes=[NodeData(index, {"label": f"node-{index}"}) for index in range(nodes)], + edges=[ + EdgeData(source, target, {"relation": "RELATED_TO", "weight": 1.0}) + for source, target in sorted(seen) + ], + ) + + +def directory_size(path: Path) -> int: + return sum(child.stat().st_size for child in path.rglob("*") if child.is_file()) + + +def run_size(nodes: int, edges: int, root: Path) -> dict[str, Any]: + import networkx as networkx # benchmark-only dependency + + result: dict[str, Any] = {"nodes": nodes, "edges": edges} + nx_path = root / f"{nodes}-{edges}.networkx.json" + + def persist_networkx(graph: Any) -> None: + payload = networkx.node_link_data(graph, edges="links") + nx_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + def networkx_ingest(): + graph = topology(networkx.Graph, nodes, edges, 42) + persist_networkx(graph) + return graph + + nx_graph, nx_ingest, _ = measure(networkx_ingest) + _, nx_reopen, nx_reopen_samples = median_measure( + lambda: networkx.node_link_graph( + json.loads(nx_path.read_text(encoding="utf-8")), edges="links" + ), + 3, + ) + store_path = root / f"{nodes}-{edges}.helix" + + def ingest(): + graph = helix_topology(nodes, edges, 42) + with HelixEmbeddedStore(store_path) as store: + store.save_generation(graph, new_state(build={"benchmark": True})) + return graph + + helix_graph, helix_ingest, _ = measure(ingest) + + def reopen(): + with HelixEmbeddedStore(store_path, read_only=True) as store: + return store.load() + + loaded, helix_reopen, helix_reopen_samples = median_measure(reopen, 3) + durable = loaded.graph + if durable.node_count != nodes or durable.edge_count != edges: + raise RuntimeError("Helix reopen count verification failed") + helix_ingest_disk = directory_size(store_path) + + update_count = max(1, nodes // 100) + + def networkx_incremental(): + for index in range(update_count): + nx_graph.nodes[index]["label"] = f"updated-node-{index}" + persist_networkx(nx_graph) + + _, nx_incremental, _ = measure(networkx_incremental) + + def helix_incremental(): + changed = GraphBuildData( + nodes=[ + NodeData( + node.id, + { + **node.attributes, + **({"label": f"updated-node-{node.id}"} if isinstance(node.id, int) and node.id < update_count else {}), + }, + ) + for node in helix_graph.nodes + ], + edges=list(helix_graph.edges), + ) + with HelixEmbeddedStore(store_path) as store: + store.save_generation(changed, new_state(build={"benchmark": True, "incremental_percent": 1})) + + _, helix_incremental, _ = measure(helix_incremental) + loaded = reopen() + durable = loaded.graph + + reader = HelixGraphReader(store_path) + reader.get() + _, helix_hot_open, helix_hot_open_samples = median_measure(reader.get, 5) + probes = [index * (nodes // 20) for index in range(20)] + _, nx_neighbors, nx_neighbor_samples = median_measure( + lambda: [tuple(nx_graph.neighbors(node)) for node in probes], 5 + ) + _, helix_neighbors, helix_neighbor_samples = median_measure( + lambda: [durable.neighbors(node) for node in probes], 5 + ) + _, nx_bfs, nx_bfs_samples = median_measure( + lambda: list(networkx.bfs_tree(nx_graph, 0, depth_limit=4)), 5 + ) + _, helix_bfs, helix_bfs_samples = median_measure( + lambda: _bounded_bfs(durable, 0, 4), 5 + ) + _, nx_shortest, nx_shortest_samples = median_measure( + lambda: [networkx.shortest_path(nx_graph, 0, 4) for _ in range(5)], 5 + ) + _, helix_shortest, helix_shortest_samples = median_measure( + lambda: [durable.shortest_path(0, 4) for _ in range(5)], 5 + ) + _, nx_louvain, nx_louvain_samples = median_measure( + lambda: networkx.community.louvain_communities(nx_graph, seed=42), 3 + ) + _, helix_leiden, helix_leiden_samples = median_measure( + lambda: durable.to_undirected().leiden().communities, 3 + ) + _, nx_centrality, nx_centrality_samples = median_measure( + lambda: networkx.betweenness_centrality(nx_graph, k=min(100, nodes), seed=42), 1 + ) + _, helix_centrality, helix_centrality_samples = median_measure( + lambda: durable.betweenness_centrality( + __import__("helixdb").BetweennessOptions(mode="sampled", sample_count=min(100, nodes), seed=42) + ), 1 + ) + _, nx_edge_centrality, nx_edge_centrality_samples = median_measure( + lambda: networkx.edge_betweenness_centrality( + nx_graph, k=min(100, nodes), seed=42 + ), 1 + ) + _, helix_edge_centrality, helix_edge_centrality_samples = median_measure( + lambda: durable.edge_betweenness_centrality( + __import__("helixdb").BetweennessOptions( + mode="sampled", sample_count=min(100, nodes), seed=42 + ) + ), 1 + ) + + nx_export_path = root / f"{nodes}-{edges}.networkx.graphml" + helix_export_path = root / f"{nodes}-{edges}.helix.graphml" + _, nx_export, _ = measure(lambda: networkx.write_graphml(nx_graph, nx_export_path)) + + def helix_export_graphml(): + from graphify.export import to_graphml + + to_graphml(durable, {}, str(helix_export_path)) + + _, helix_export, _ = measure(helix_export_graphml) + + def concurrent_readers(): + with ThreadPoolExecutor(max_workers=4) as pool: + return list(pool.map(lambda _: reopen().graph.node_count, range(8))) + + _, helix_concurrency, _ = measure(concurrent_readers) + nx_memory = isolated_ingest_memory("networkx", nodes, edges) + helix_memory = isolated_ingest_memory("helix", nodes, edges) + result["networkx"] = { + "ingest_seconds": nx_ingest, + "ingest_runs": 1, + "reopen_seconds": nx_reopen, + "reopen_samples_seconds": nx_reopen_samples, + "neighbor_20_queries_seconds": nx_neighbors, + "neighbor_samples_seconds": nx_neighbor_samples, + "bfs_seconds": nx_bfs, + "bfs_samples_seconds": nx_bfs_samples, + "shortest_path_5_queries_seconds": nx_shortest, + "shortest_path_samples_seconds": nx_shortest_samples, + "community_detection": "Louvain (production NetworkX comparator)", + "community_seconds": nx_louvain, + "community_samples_seconds": nx_louvain_samples, + "node_betweenness_seconds": nx_centrality, + "node_betweenness_samples_seconds": nx_centrality_samples, + "edge_betweenness_seconds": nx_edge_centrality, + "edge_betweenness_samples_seconds": nx_edge_centrality_samples, + "betweenness_mode": f"sampled ({min(100, nodes)} sources, seed 42)", + "incremental_1pct_seconds": nx_incremental, + "graphml_export_seconds": nx_export, + "peak_rss_delta_bytes": nx_memory, + "disk_bytes": nx_path.stat().st_size, + } + result["helix"] = { + "ingest_seconds": helix_ingest, + "ingest_runs": 1, + "reopen_seconds": helix_reopen, + "reopen_samples_seconds": helix_reopen_samples, + "hot_open_seconds": helix_hot_open, + "hot_open_samples_seconds": helix_hot_open_samples, + "neighbor_20_queries_seconds": helix_neighbors, + "neighbor_samples_seconds": helix_neighbor_samples, + "bfs_seconds": helix_bfs, + "bfs_samples_seconds": helix_bfs_samples, + "shortest_path_5_queries_seconds": helix_shortest, + "shortest_path_samples_seconds": helix_shortest_samples, + "community_detection": "weighted Leiden (Graphify production)", + "community_seconds": helix_leiden, + "community_samples_seconds": helix_leiden_samples, + "node_betweenness_seconds": helix_centrality, + "node_betweenness_samples_seconds": helix_centrality_samples, + "edge_betweenness_seconds": helix_edge_centrality, + "edge_betweenness_samples_seconds": helix_edge_centrality_samples, + "betweenness_mode": f"sampled ({min(100, nodes)} sources, seed 42)", + "incremental_1pct_seconds": helix_incremental, + "graphml_export_seconds": helix_export, + "eight_concurrent_reopens_seconds": helix_concurrency, + "peak_rss_delta_bytes": helix_memory, + "disk_after_ingest_bytes": helix_ingest_disk, + "disk_after_update_bytes": directory_size(store_path), + } + return result + + +def acceptance_gates(results: list[dict[str, Any]]) -> dict[str, Any]: + """Evaluate the release thresholds and retain every measured comparison.""" + checks: list[dict[str, Any]] = [] + + def check(name: str, actual: float, limit: float, comparison: str = "<=") -> None: + passed = actual <= limit if comparison == "<=" else actual >= limit + checks.append({ + "name": name, + "actual": actual, + "limit": limit, + "comparison": comparison, + "passed": passed, + }) + + for row in results: + label = f"{row['nodes']}/{row['edges']}" + helix = row["helix"] + networkx = row["networkx"] + cold_limit = 0.5 if row["nodes"] == 5_000 else 3.0 + hot_limit = 0.100 if row["nodes"] == 5_000 else 0.500 + check( + f"{label} ingest vs v8", + helix["ingest_seconds"] / networkx["ingest_seconds"], + 2.0, + ) + check( + f"{label} 1% update vs v8", + helix["incremental_1pct_seconds"] / networkx["incremental_1pct_seconds"], + 2.0, + ) + check(f"{label} cold open seconds", helix["reopen_seconds"], cold_limit) + check( + f"{label} cold open vs v8", + helix["reopen_seconds"] / networkx["reopen_seconds"], + 5.0, + ) + check( + f"{label} active store vs v8", + helix["disk_after_ingest_bytes"] / networkx["disk_bytes"], + 3.0, + ) + check( + f"{label} post-update store vs v8", + helix["disk_after_update_bytes"] / networkx["disk_bytes"], + 3.0, + ) + if row["nodes"] == 20_000: + check( + f"{label} clustering speedup", + networkx["community_seconds"] / helix["community_seconds"], + 3.0, + ">=", + ) + check( + f"{label} node centrality speedup", + networkx["node_betweenness_seconds"] / helix["node_betweenness_seconds"], + 3.0, + ">=", + ) + check( + f"{label} edge centrality speedup", + networkx["edge_betweenness_seconds"] / helix["edge_betweenness_seconds"], + 3.0, + ">=", + ) + check(f"{label} hot_open_seconds", helix["hot_open_seconds"], hot_limit) + for metric, baseline_metric in ( + ("neighbor_20_queries_seconds", "neighbor_20_queries_seconds"), + ("bfs_seconds", "bfs_seconds"), + ("shortest_path_5_queries_seconds", "shortest_path_5_queries_seconds"), + ): + check(f"{label} {metric}", helix[metric], hot_limit) + check( + f"{label} {metric} vs v8", + helix[metric] / networkx[baseline_metric], + 2.0, + ) + return {"passed": all(item["passed"] for item in checks), "checks": checks} + + +def _bounded_bfs(graph, start: Any, depth: int) -> list[Any]: + visited = {start} + frontier = {start} + for _ in range(depth): + frontier = {neighbor for node in frontier for neighbor in graph.neighbors(node)} - visited + visited.update(frontier) + return list(visited) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--out", type=Path, default=Path("benchmarks/helix-vs-networkx.json")) + parser.add_argument("--memory-backend", choices=("networkx", "helix")) + parser.add_argument("--memory-nodes", type=int) + parser.add_argument("--memory-edges", type=int) + parser.add_argument("--check-gates", action="store_true") + args = parser.parse_args() + if args.memory_backend: + if args.memory_nodes is None or args.memory_edges is None: + parser.error("--memory-backend requires --memory-nodes and --memory-edges") + print(memory_only(args.memory_backend, args.memory_nodes, args.memory_edges)) + return + root = Path(tempfile.mkdtemp(prefix="graphify-benchmark-")) + try: + results = [run_size(nodes, edges, root) for nodes, edges in SIZES] + report = { + "helix_revision": "0.2.0b3", + "python": platform.python_version(), + "platform": platform.platform(), + "pid": os.getpid(), + "results": results, + "acceptance": acceptance_gates(results), + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps(report, indent=2)) + if args.check_gates and not report["acceptance"]["passed"]: + raise SystemExit(1) + finally: + shutil.rmtree(root, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/parity-networkx.json b/benchmarks/parity-networkx.json new file mode 100644 index 000000000..2a1684130 --- /dev/null +++ b/benchmarks/parity-networkx.json @@ -0,0 +1,15 @@ +{ + "helix_revision": "0.2.0b3", + "networkx_version": "3.6.1", + "checks": { + "directed_path": true, + "bfs": true, + "dfs": true, + "node_betweenness": true, + "edge_betweenness": true, + "louvain": true, + "layout": true, + "transformations": true + }, + "all_passed": true +} diff --git a/benchmarks/parity_networkx.py b/benchmarks/parity_networkx.py new file mode 100644 index 000000000..2a02d4072 --- /dev/null +++ b/benchmarks/parity_networkx.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Isolated behavioral parity checks; NetworkX is benchmark-only.""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile + +import networkx + +from graphify.helix.model import EdgeData, GraphBuildData, NodeData +from graphify.helix.native import native_backend_info +from graphify.helix.persistence import HelixEmbeddedStore +from graphify.helix.state import new_state + + +def load_native(build: GraphBuildData, root: Path): + with HelixEmbeddedStore(root / "graph.helix") as store: + store.save_generation(build, new_state()) + with HelixEmbeddedStore(root / "graph.helix", read_only=True) as store: + return store.load().graph + + +def close_scores(left: dict, right: dict, tolerance: float = 1e-9) -> bool: + return left.keys() == right.keys() and all( + abs(left[key] - right[key]) <= tolerance for key in left + ) + + +def main() -> None: + with tempfile.TemporaryDirectory(prefix="graphify-parity-") as temporary: + root = Path(temporary) + nodes = [NodeData("a"), NodeData(2), NodeData(b"c")] + edges = [ + EdgeData("a", 2, {"relation": "calls", "weight": 1.0}), + EdgeData(2, b"c", {"relation": "calls", "weight": 1.0}), + ] + native = load_native(GraphBuildData(kind="digraph", nodes=nodes, edges=edges), root) + reference = networkx.DiGraph() + reference.add_edge("a", 2, weight=1.0) + reference.add_edge(2, b"c", weight=1.0) + + from helixdb import TraversalOptions + + checks = { + "directed_path": native.shortest_path("a", b"c", direction="out").node_ids + == tuple(networkx.shortest_path(reference, "a", b"c")), + "bfs": {visit.node_id for visit in native.traverse(TraversalOptions( + seeds=("a",), max_depth=10, strategy="breadth_first", direction="out" + )).visits} == set(networkx.bfs_tree(reference, "a")), + "dfs": {visit.node_id for visit in native.traverse(TraversalOptions( + seeds=("a",), max_depth=10, strategy="depth_first", direction="out" + )).visits} == set(networkx.dfs_tree(reference, "a")), + } + native_node_scores = {row.node_id: row.score for row in native.betweenness_centrality()} + checks["node_betweenness"] = close_scores( + native_node_scores, networkx.betweenness_centrality(reference) + ) + native_edge_scores = { + (row.source, row.target): row.score + for row in native.edge_betweenness_centrality() + } + checks["edge_betweenness"] = close_scores( + native_edge_scores, networkx.edge_betweenness_centrality(reference) + ) + + community_nodes = [NodeData(name) for name in "abcdef"] + community_edges = [ + EdgeData(u, v, {"relation": "related", "weight": 1.0}) + for group in ("abc", "def") + for index, u in enumerate(group) + for v in group[index + 1 :] + ] + communities = load_native( + GraphBuildData(nodes=community_nodes, edges=community_edges), root / "communities" + ) + native_partition = {frozenset(row.node_ids) for row in communities.louvain_communities().communities} + nx_community = networkx.Graph() + nx_community.add_edges_from((edge.source, edge.target) for edge in community_edges) + nx_partition = { + frozenset(group) + for group in networkx.community.louvain_communities(nx_community, seed=42) + } + checks["louvain"] = native_partition == nx_partition + checks["layout"] = len(communities.spring_layout()) == 6 + checks["transformations"] = ( + communities.induced_subgraph(["a", "b"]).edge_count == 1 + and communities.relabel({"a": ("renamed", "a")}).contains_node(("renamed", "a")) + and communities.to_directed().directed + ) + + report = { + "helix_revision": native_backend_info().embedded_version, + "networkx_version": networkx.__version__, + "checks": checks, + "all_passed": all(checks.values()), + } + print(json.dumps(report, indent=2)) + if not report["all_passed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/report-erpnext-2026-07-19.json b/benchmarks/report-erpnext-2026-07-19.json new file mode 100644 index 000000000..b0b7caf4e --- /dev/null +++ b/benchmarks/report-erpnext-2026-07-19.json @@ -0,0 +1,131 @@ +{ + "qualification_date": "2026-07-19", + "report_source": "helix_report.pdf (provided artifact)", + "method": { + "command": "graphify update --no-cluster", + "llm_environment_removed": [ + "ANTHROPIC_API_KEY", + "MOONSHOT_API_KEY", + "OLLAMA_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY", + "AZURE_OPENAI_API_KEY", + "AWS_PROFILE", + "AWS_REGION", + "AWS_DEFAULT_REGION" + ], + "timing": "/usr/bin/time -lp", + "runs_sequential": true, + "peak_rss_blocking": false + }, + "versions": { + "python": "3.12.13", + "platform": "macOS-26.5.2-arm64-arm-64bit", + "helix_db": "0.2.0b3", + "helix_db_embedded": "0.2.0b3", + "graphify_v8_commit": "edec9eabeceeae6aa2375eddb3835efa1a32c0a3", + "candidate_base_commit": "17169c0ebcc2e64f54e2504ee2270e6a1cb1dc92" + }, + "corpus": { + "name": "erpnext", + "repository": "frappe/erpnext", + "commit": "ea5c648ab04a2b30c5c238f6cb299c4237ff1c1e" + }, + "topology": { + "v8_serialized_nodes": 24889, + "v8_serialized_edges": 59142, + "v8_loaded_nodes": 25443, + "v8_loaded_edges": 59142, + "helix_loaded_nodes": 25443, + "helix_loaded_edges": 59142, + "parity": true, + "note": "The v8 no-cluster payload relies on multigraph rehydration to create 554 external endpoint stubs. The candidate now preserves those stubs and every relation-distinct parallel edge natively." + }, + "fresh_build": { + "v8_seconds": 41.30, + "helix_seconds": 99.15, + "helix_over_v8": 2.4007263922518165, + "limit": 2.0, + "passed": false, + "v8_peak_rss_bytes": 1050017792, + "helix_peak_rss_bytes": 1630797824 + }, + "immediate_update": { + "v8_seconds": 40.13, + "helix_seconds": 46.65, + "helix_over_v8": 1.162471966110142, + "limit": 2.0, + "passed": true, + "helix_topology_unchanged": true, + "helix_peak_rss_bytes": 2554691584 + }, + "cold_open": { + "v8_samples_seconds": [0.317707, 0.328374, 0.343941], + "helix_samples_seconds": [7.904627, 7.948525, 8.007312], + "v8_median_seconds": 0.328374, + "helix_median_seconds": 7.948525, + "helix_over_v8": 24.205707516429438, + "absolute_limit_seconds": 3.0, + "relative_limit": 5.0, + "passed": false + }, + "active_store": { + "v8_bytes": 34901442, + "helix_bytes": 360832137, + "helix_over_v8": 10.338602542553973, + "limit": 3.0, + "passed": false, + "retain_rollback": false + }, + "representative_query": { + "question": "payment entry reconciliation", + "depth": 2, + "token_budget": 500, + "v8_warm_samples_seconds": [ + 0.900777, + 0.054018, + 0.050583, + 0.049234, + 0.048877 + ], + "helix_warm_samples_seconds": [ + 4.008484, + 3.529874, + 3.519547, + 3.522045, + 3.527935 + ], + "v8_steady_median_seconds": 0.0499085, + "helix_steady_median_seconds": 3.52499, + "absolute_limit_seconds": 0.5, + "relative_limit": 2.0, + "passed": false, + "note": "This is an operational latency probe, not a gold-recall substitute." + }, + "unavailable_report_inputs": { + "corpora": ["comfywerk", "agent", "backend", "passport"], + "gold_queries": "graphify-bench/queries/*.json", + "harnesses": ["bench_helix.py", "bench_inprocess.py"], + "raw_results": "helix_bench_results.json", + "searched_locations": [ + "workspace repositories", + "provided downloads", + "temporary work directories", + "GitHub repository/code search", + "PR #1993 comments and reviews", + "PDF link annotations" + ], + "effect": "The report's exact five-corpus gold-recall table cannot be reproduced or claimed passing without these non-redistributed inputs." + }, + "merge_ready": false, + "blocking_gates": [ + "fresh build exceeds 2x v8", + "cold open exceeds both absolute and relative limits", + "active store exceeds 3x v8", + "steady query exceeds both absolute and relative limits", + "exact report corpora, gold queries, harnesses, and raw results are unavailable", + "public helix-db-embedded 0.2.0b3 has no win_amd64 wheel" + ] +} diff --git a/benchmarks/requirements-networkx.txt b/benchmarks/requirements-networkx.txt new file mode 100644 index 000000000..f8efd2cab --- /dev/null +++ b/benchmarks/requirements-networkx.txt @@ -0,0 +1,2 @@ +# Isolated comparison environment only; never install in Graphify production. +networkx==3.6.1 diff --git a/docs/how-it-works.md b/docs/how-it-works.md index e0e6e5275..573c3cdb2 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -64,7 +64,7 @@ On a mixed corpus (Karpathy repos + 5 papers + 4 images, 52 files): **71.5x fewe Token reduction scales with corpus size. Six files already fits in a context window — the graph value there is structural clarity, not compression. At 52 files the savings compound quickly. -Each `worked/` folder in the repo has the raw input files and actual output (`GRAPH_REPORT.md`, `graph.json`) so you can run it yourself and verify. +Each `worked/` folder in the repo has raw input files and a report so you can run the current native build yourself and verify it. --- @@ -76,13 +76,13 @@ Code files are extracted in parallel using `ProcessPoolExecutor` — bypasses Py ## SHA256 cache -Every extracted file is fingerprinted by content hash. Re-runs skip unchanged files entirely — only new or modified files go through extraction again. The cache lives in `graphify-out/cache/`. +Every extracted file is fingerprinted by content hash. Re-runs skip unchanged files entirely — only new or modified files go through extraction again. Hashes and extraction cache entries live in the active Helix generation. --- ## The graph format -The output `graph.json` uses NetworkX's node-link format. Each node has: +The embedded `graphify-out/graph.helix` store is the sole runtime graph. Each native node has: - `id` — stable identifier - `label` — human-readable name - `file_type` — `code`, `document`, `paper`, `image`, `rationale` @@ -98,4 +98,4 @@ Each edge has: - `confidence_score` — float (INFERRED only) - `source_file` — where the relationship was found -Hyperedges (group relationships connecting 3+ nodes) live in `G.graph["hyperedges"]`. +Hyperedges (group relationships connecting 3+ nodes) are stored as generation-scoped native metadata. diff --git a/docs/node-summaries-rfc.md b/docs/node-summaries-rfc.md index af191f3f8..8ab2d37b9 100644 --- a/docs/node-summaries-rfc.md +++ b/docs/node-summaries-rfc.md @@ -5,7 +5,7 @@ summaries for AI coding agents. ## Problem -`graph.json` gives agents graph structure, source files, node labels, and +The native Helix graph gives agents graph structure, source files, node labels, and relationships. That helps avoid reading an entire repository, but agents still often need to inspect raw files just to answer a basic navigation question: @@ -84,7 +84,7 @@ or every symbol in the file. Those details already belong in the graph and can be fetched through `graphify explain`, `graphify query`, or direct file reads when needed. -## Option A: `summary` attribute in `graph.json` +## Option A: `summary` property on native nodes Add an optional `summary` field to file-level nodes: @@ -107,8 +107,8 @@ graphify explain "extract.py" Pros: -- Single artifact for graph consumers. -- Matches NetworkX node attributes and existing node metadata. +- One generation-safe store for graph consumers. +- Matches existing native node properties and metadata. - Easy for `explain`, `serve`, visualizers, and MCP tools to consume. - No sidecar freshness or node-ID join logic. @@ -116,12 +116,11 @@ Cons: - Adds text to the core graph artifact. - Expands the graph schema surface. -- Consumers that dump all of `graph.json` into an LLM context would pay for all - summaries at once. +- Consumers that request every node would pay for all summaries at once. -## Option B: sidecar `node-summaries.json` +## Option B: generation-scoped summary records -Write summaries to a separate artifact keyed by node ID: +Write summaries as generation-scoped native records keyed by node ID: ```json { @@ -146,15 +145,15 @@ graphify explain "extract.py" Pros: -- Keeps `graph.json` lean and topology-focused. +- Keeps topology records lean and focused. - Makes summaries clearly optional. - Can be regenerated independently. - Provides a natural place for future generator metadata. Cons: -- Adds a second artifact that consumers must discover and load. -- Introduces freshness and synchronization questions. +- Adds a second native record family that consumers must query. +- Requires transactional activation with the topology generation. - Every consumer that wants summaries must join by node ID. ## Suggested first implementation once storage is chosen @@ -177,8 +176,7 @@ Cons: ## Questions for maintainers and users -1. Should graphify prefer one artifact (`graph.json`) or keep generated text in a - sidecar? +1. Should graphify prefer node properties or generation-scoped summary records? 2. Should deterministic file-level summaries be generated during graph creation, or only through an explicit command such as `graphify summarize`? 3. Is `summary` the right term, or would `synopsis` better communicate a short, diff --git a/docs/superpowers/plans/2026-05-04-incremental-updates-dedup.md b/docs/superpowers/plans/2026-05-04-incremental-updates-dedup.md deleted file mode 100644 index 488591b8d..000000000 --- a/docs/superpowers/plans/2026-05-04-incremental-updates-dedup.md +++ /dev/null @@ -1,1143 +0,0 @@ -# Incremental Updates + Entity Deduplication Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add semantic cache + incremental graph updates to `graphify extract`, and add a new `graphify/dedup.py` module that runs MinHash/LSH + Jaro-Winkler entity deduplication before clustering on every run. - -**Architecture:** Two independent features wired into the same pipeline: (1) `__main__.py` extract block uses `check_semantic_cache`/`save_semantic_cache` + `detect_incremental` + `build_merge` for incremental runs; (2) new `graphify/dedup.py` implements the full dedup pipeline called from `build.py` after graph construction and before `cluster()`. - -**Tech Stack:** Python 3.10+, `datasketch` (MinHash/LSH), `rapidfuzz` (Jaro-Winkler), `networkx` (union-find via `nx.utils.UnionFind`), existing `graphify.cache`, `graphify.detect`, `graphify.build`. - ---- - -## File Map - -| File | Action | Responsibility | -|---|---|---| -| `graphify/dedup.py` | **Create** | Full dedup pipeline: entropy gate → MinHash/LSH → Jaro-Winkler → community boost → union-find merge | -| `graphify/build.py` | **Modify** | Call `deduplicate_entities()` from `build()` and `build_merge()`; wire dormant `deduplicate_by_label` | -| `graphify/__main__.py` | **Modify** | Semantic cache wrapping, incremental mode auto-detection, `build_merge` swap, manifest save, `--dedup-llm` flag | -| `pyproject.toml` | **Modify** | Add `datasketch`, `rapidfuzz` to base dependencies | -| `tests/test_dedup.py` | **Create** | Unit + integration tests for dedup pipeline | -| `tests/test_incremental.py` | **Create** | Integration tests for incremental extract (cache hits, manifest, prune) | - ---- - -## Task 1: Add `datasketch` and `rapidfuzz` to dependencies - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Add deps to pyproject.toml** - -Open `pyproject.toml`. The `dependencies` list ends around line 38. Add two entries: - -```toml -dependencies = [ - "networkx", - "datasketch", - "rapidfuzz", - "tree-sitter>=0.23.0", - # ... rest unchanged -] -``` - -- [ ] **Step 2: Install into venv** - -```bash -cd /home/safi/graphify -venv/bin/pip install datasketch rapidfuzz -q -``` - -Expected: both install without errors. - -- [ ] **Step 3: Verify import** - -```bash -venv/bin/python -c "from datasketch import MinHash, MinHashLSH; from rapidfuzz import fuzz; print('ok')" -``` - -Expected: `ok` - -- [ ] **Step 4: Commit** - -```bash -git add pyproject.toml -git commit -m "Add datasketch and rapidfuzz as base dependencies" -``` - ---- - -## Task 2: Create `graphify/dedup.py` — entropy gate + MinHash/LSH + Jaro-Winkler - -**Files:** -- Create: `graphify/dedup.py` -- Create: `tests/test_dedup.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/test_dedup.py`: - -```python -"""Tests for graphify/dedup.py entity deduplication pipeline.""" -from __future__ import annotations -import pytest -from graphify.dedup import deduplicate_entities, _entropy, _shingles - - -# ── entropy gate ───────────────────────────────────────────────────────────── - -def test_entropy_short_label_low(): - assert _entropy("AI") < 2.5 - -def test_entropy_normal_label_high(): - assert _entropy("AuthenticationManager") >= 2.5 - -def test_entropy_empty_string(): - assert _entropy("") == 0.0 - - -# ── shingles ───────────────────────────────────────────────────────────────── - -def test_shingles_produces_trigrams(): - s = _shingles("hello") - assert "hel" in s - assert "ell" in s - assert "llo" in s - -def test_shingles_short_string(): - # strings shorter than 3 chars return single shingle of the string itself - assert _shingles("ab") == {"ab"} - - -# ── full pipeline ───────────────────────────────────────────────────────────── - -def _make_nodes(*labels): - return [{"id": label.lower().replace(" ", "_"), "label": label, "source_file": "test.md"} for label in labels] - -def _make_edges(src, tgt, relation="relates_to"): - return [{"source": src, "target": tgt, "relation": relation}] - - -def test_exact_duplicates_merged(): - nodes = _make_nodes("UserService", "userservice", "User Service") - edges = [] - result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={}) - labels = {n["label"] for n in result_nodes} - # All three are the same concept — only one survives - assert len(result_nodes) == 1 - - -def test_typo_merged(): - # "GraphExtractor" vs "Graph Extractor" — Jaro-Winkler >= 0.92 - nodes = _make_nodes("GraphExtractor", "Graph Extractor") - edges = [] - result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) - assert len(result_nodes) == 1 - - -def test_unrelated_not_merged(): - nodes = _make_nodes("UserService", "OrderService") - edges = [] - result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) - assert len(result_nodes) == 2 - - -def test_short_low_entropy_not_merged(): - # "AI" and "ML" are low-entropy — entropy gate skips them - nodes = _make_nodes("AI", "ML") - edges = [] - result_nodes, _ = deduplicate_entities(nodes, edges, communities={}) - assert len(result_nodes) == 2 - - -def test_edges_rewired_after_merge(): - nodes = _make_nodes("GraphExtractor", "Graph Extractor", "Parser") - # edge from loser to Parser should be rewired to winner - edges = [{"source": "graph_extractor", "target": "parser", "relation": "uses"}] - result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={}) - assert len(result_nodes) == 2 # merged + Parser - # edge should still exist (rewired to winner) - assert len(result_edges) == 1 - - -def test_self_loops_dropped_after_merge(): - # If both endpoints of an edge get merged into same node, drop the edge - nodes = _make_nodes("GraphExtractor", "Graph Extractor") - edges = [{"source": "graphextractor", "target": "graph_extractor", "relation": "same"}] - _, result_edges = deduplicate_entities(nodes, edges, communities={}) - assert result_edges == [] - - -def test_community_boost_aids_merge(): - # Two nodes in same community with score in 0.75-0.85 zone get boosted - # This is a structural test — use labels that would score ~0.80 without boost - # We verify that with communities set, they merge, without they don't - # Use labels that Jaro-Winkler scores ~0.88 (borderline) - nodes = _make_nodes("AuthManager", "Auth Manager") - edges = [] - # Same community → boost → merge - communities = {"authmanager": 1, "auth_manager": 1} - result_with, _ = deduplicate_entities(nodes, edges, communities=communities) - # Different community → no boost - communities_diff = {"authmanager": 1, "auth_manager": 2} - result_without, _ = deduplicate_entities(nodes, edges, communities=communities_diff) - assert len(result_with) <= len(result_without) - - -def test_empty_inputs(): - result_nodes, result_edges = deduplicate_entities([], [], communities={}) - assert result_nodes == [] - assert result_edges == [] - - -def test_single_node_no_crash(): - nodes = _make_nodes("UserService") - result_nodes, _ = deduplicate_entities(nodes, [], communities={}) - assert len(result_nodes) == 1 -``` - -- [ ] **Step 2: Run tests — verify they all fail** - -```bash -cd /home/safi/graphify -venv/bin/python -m pytest tests/test_dedup.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all fail with `ModuleNotFoundError: No module named 'graphify.dedup'` - -- [ ] **Step 3: Create `graphify/dedup.py`** - -```python -"""Entity deduplication pipeline for graphify knowledge graphs. - -Pipeline: exact normalization → entropy gate → MinHash/LSH blocking → -Jaro-Winkler verification → same-community boost → union-find merge. -""" -from __future__ import annotations -import math -import re -from collections import defaultdict -from typing import Any - -from datasketch import MinHash, MinHashLSH -from rapidfuzz import fuzz - - -# ── helpers ─────────────────────────────────────────────────────────────────── - -def _norm(label: str) -> str: - """Lowercase + collapse non-alphanumeric runs to space.""" - return re.sub(r"[^a-z0-9]+", " ", label.lower()).strip() - - -def _entropy(label: str) -> float: - """Shannon entropy in bits/char of the normalised label.""" - s = _norm(label) - if not s: - return 0.0 - freq: dict[str, int] = defaultdict(int) - for ch in s: - freq[ch] += 1 - n = len(s) - return -sum((c / n) * math.log2(c / n) for c in freq.values()) - - -def _shingles(text: str, k: int = 3) -> set[str]: - """Return k-gram character shingles of text.""" - if len(text) < k: - return {text} - return {text[i : i + k] for i in range(len(text) - k + 1)} - - -def _make_minhash(text: str, num_perm: int = 128) -> MinHash: - m = MinHash(num_perm=num_perm) - for shingle in _shingles(text): - m.update(shingle.encode("utf-8")) - return m - - -# ── union-find ──────────────────────────────────────────────────────────────── - -class _UF: - def __init__(self) -> None: - self._parent: dict[str, str] = {} - - def find(self, x: str) -> str: - self._parent.setdefault(x, x) - while self._parent[x] != x: - self._parent[x] = self._parent[self._parent[x]] - x = self._parent[x] - return x - - def union(self, x: str, y: str) -> None: - self._parent.setdefault(x, x) - self._parent.setdefault(y, y) - rx, ry = self.find(x), self.find(y) - if rx != ry: - self._parent[ry] = rx - - def components(self) -> dict[str, list[str]]: - groups: dict[str, list[str]] = defaultdict(list) - for x in self._parent: - groups[self.find(x)].append(x) - return dict(groups) - - -# ── main entry point ────────────────────────────────────────────────────────── - -_ENTROPY_THRESHOLD = 2.5 -_LSH_THRESHOLD = 0.7 -_JW_THRESHOLD = 92.0 # rapidfuzz returns 0-100 -_COMMUNITY_BOOST = 5.0 # added to score when both nodes share community -_MERGE_THRESHOLD = 92.0 # final threshold after boost -_NUM_PERM = 128 -_CHUNK_SUFFIX = re.compile(r"_c\d+$") - - -def deduplicate_entities( - nodes: list[dict], - edges: list[dict], - *, - communities: dict[str, int], -) -> tuple[list[dict], list[dict]]: - """Deduplicate near-identical entities in a knowledge graph. - - Args: - nodes: list of node dicts with at minimum {"id": str, "label": str} - edges: list of edge dicts with {"source": str, "target": str, ...} - communities: mapping of node_id -> community_id (from cluster()) - - Returns: - (deduped_nodes, deduped_edges) with edges rewired to survivors - """ - if len(nodes) <= 1: - return nodes, edges - - # ── pass 1: exact normalization (always runs) ───────────────────────────── - norm_to_nodes: dict[str, list[dict]] = defaultdict(list) - for node in nodes: - key = _norm(node.get("label", node.get("id", ""))) - if key: - norm_to_nodes[key].append(node) - - uf = _UF() - for key, group in norm_to_nodes.items(): - if len(group) > 1: - winner = _pick_winner(group) - for node in group: - uf.union(winner["id"], node["id"]) - - exact_merges = sum(len(g) - 1 for g in norm_to_nodes.values() if len(g) > 1) - - # ── pass 2: MinHash/LSH + Jaro-Winkler (high-entropy nodes only) ───────── - # Build candidate set: one representative per exact-norm group - candidates: list[dict] = [] - seen_norms: set[str] = set() - for node in nodes: - key = _norm(node.get("label", node.get("id", ""))) - if key and key not in seen_norms: - seen_norms.add(key) - if _entropy(node.get("label", "")) >= _ENTROPY_THRESHOLD: - candidates.append(node) - - fuzzy_merges = 0 - if len(candidates) >= 2: - lsh = MinHashLSH(threshold=_LSH_THRESHOLD, num_perm=_NUM_PERM) - minhashes: dict[str, MinHash] = {} - - for node in candidates: - norm_label = _norm(node.get("label", node.get("id", ""))) - m = _make_minhash(norm_label) - minhashes[node["id"]] = m - try: - lsh.insert(node["id"], m) - except ValueError: - pass # duplicate key in LSH — already inserted - - for node in candidates: - node_id = node["id"] - norm_label = _norm(node.get("label", node.get("id", ""))) - neighbors = lsh.query(minhashes[node_id]) - - for neighbor_id in neighbors: - if neighbor_id == node_id: - continue - if uf.find(node_id) == uf.find(neighbor_id): - continue # already merged - - # Find the neighbour node - neighbor = next((n for n in candidates if n["id"] == neighbor_id), None) - if neighbor is None: - continue - - neighbor_norm = _norm(neighbor.get("label", neighbor.get("id", ""))) - score = fuzz.jaro_winkler_similarity(norm_label, neighbor_norm) * 100 - - # Same-community boost - c1 = communities.get(node_id) - c2 = communities.get(neighbor_id) - if c1 is not None and c2 is not None and c1 == c2: - score += _COMMUNITY_BOOST - - if score >= _MERGE_THRESHOLD: - all_nodes_in_group = norm_to_nodes.get(norm_label, [node]) + \ - norm_to_nodes.get(neighbor_norm, [neighbor]) - winner = _pick_winner(all_nodes_in_group) - uf.union(winner["id"], node_id) - uf.union(winner["id"], neighbor_id) - fuzzy_merges += 1 - - # ── build remap table from union-find components ────────────────────────── - components = uf.components() - remap: dict[str, str] = {} - surviving_ids: set[str] = set() - - for root, members in components.items(): - if len(members) == 1: - surviving_ids.add(root) - continue - group_nodes = [n for n in nodes if n["id"] in members] - winner = _pick_winner(group_nodes) if group_nodes else {"id": root} - winner_id = winner["id"] - surviving_ids.add(winner_id) - for member in members: - if member != winner_id: - remap[member] = winner_id - - # ── apply remap ─────────────────────────────────────────────────────────── - if not remap: - return nodes, edges - - total = len(remap) - msg = f"[graphify] Deduplicated {total} node(s)" - if exact_merges: - msg += f" ({exact_merges} exact" - if fuzzy_merges: - msg += f", {fuzzy_merges} fuzzy" - msg += ")" - print(msg + ".", flush=True) - - deduped_nodes = [n for n in nodes if n["id"] not in remap] - deduped_edges = [] - for edge in edges: - e = dict(edge) - e["source"] = remap.get(e["source"], e["source"]) - e["target"] = remap.get(e["target"], e["target"]) - if e["source"] != e["target"]: - deduped_edges.append(e) - - return deduped_nodes, deduped_edges - - -def _pick_winner(nodes: list[dict]) -> dict: - """Pick the canonical survivor: prefer no chunk suffix, then shorter ID.""" - if not nodes: - raise ValueError("Cannot pick winner from empty list") - def _score(n: dict) -> tuple[int, int]: - has_suffix = bool(_CHUNK_SUFFIX.search(n["id"])) - return (1 if has_suffix else 0, len(n["id"])) - return min(nodes, key=_score) -``` - -- [ ] **Step 4: Run tests — verify they pass** - -```bash -cd /home/safi/graphify -venv/bin/python -m pytest tests/test_dedup.py -v --tb=short 2>&1 | tail -30 -``` - -Expected: all tests pass. - -- [ ] **Step 5: Run full suite — no regressions** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: same pass count as before (532 passed, 5 failed SQL). - -- [ ] **Step 6: Commit** - -```bash -git add graphify/dedup.py tests/test_dedup.py -git commit -m "Add graphify/dedup.py: entropy gate + MinHash/LSH + Jaro-Winkler entity deduplication" -``` - ---- - -## Task 3: Wire dedup into `build.py` - -**Files:** -- Modify: `graphify/build.py` (lines 119-137 `build()`, lines 191-244 `build_merge()`) - -- [ ] **Step 1: Write failing test** - -Add to `tests/test_dedup.py`: - -```python -def test_build_calls_dedup(): - """build() should deduplicate near-identical nodes across extractions.""" - from graphify.build import build - chunk1 = { - "nodes": [{"id": "graphextractor", "label": "GraphExtractor", "source_file": "a.py"}], - "edges": [], - } - chunk2 = { - "nodes": [{"id": "graph_extractor", "label": "Graph Extractor", "source_file": "b.py"}], - "edges": [], - } - G = build([chunk1, chunk2]) - # Should have merged to 1 node - assert G.number_of_nodes() == 1 -``` - -- [ ] **Step 2: Run — verify it fails** - -```bash -venv/bin/python -m pytest tests/test_dedup.py::test_build_calls_dedup -v --tb=short -``` - -Expected: FAIL — two separate nodes exist (no dedup wired yet). - -- [ ] **Step 3: Modify `build()` in `build.py`** - -Find `build()` at line 119. Current: - -```python -def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph: - ... - combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - for ext in extractions: - combined["nodes"].extend(ext.get("nodes", [])) - combined["edges"].extend(ext.get("edges", [])) - combined["hyperedges"].extend(ext.get("hyperedges", [])) - combined["input_tokens"] += ext.get("input_tokens", 0) - combined["output_tokens"] += ext.get("output_tokens", 0) - return build_from_json(combined, directed=directed) -``` - -Replace with: - -```python -def build(extractions: list[dict], *, directed: bool = False, dedup: bool = True) -> nx.Graph: - """Merge multiple extraction results into one graph. - - directed=True produces a DiGraph. dedup=True (default) runs entity - deduplication before building the NetworkX graph. - """ - from graphify.dedup import deduplicate_entities - combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - for ext in extractions: - combined["nodes"].extend(ext.get("nodes", [])) - combined["edges"].extend(ext.get("edges", [])) - combined["hyperedges"].extend(ext.get("hyperedges", [])) - combined["input_tokens"] += ext.get("input_tokens", 0) - combined["output_tokens"] += ext.get("output_tokens", 0) - if dedup and combined["nodes"]: - combined["nodes"], combined["edges"] = deduplicate_entities( - combined["nodes"], combined["edges"], communities={} - ) - return build_from_json(combined, directed=directed) -``` - -- [ ] **Step 4: Modify `build_merge()` signature in `build.py`** - -Find `build_merge()` at line 191. Update signature and the internal `build()` call: - -```python -def build_merge( - new_chunks: list[dict], - graph_path: str | Path = "graphify-out/graph.json", - prune_sources: list[str] | None = None, - *, - directed: bool = False, - dedup: bool = True, -) -> nx.Graph: -``` - -Inside `build_merge`, find the line `G = build(all_chunks, directed=directed)` (around line 222) and replace with: - -```python - G = build(all_chunks, directed=directed, dedup=dedup) -``` - -Also update the safety-check block (around lines 235-242). When `dedup=True` or `prune_sources` is set, the graph can legitimately shrink — skip the shrink guard: - -```python - if graph_path.exists() and not dedup and not prune_sources: - existing_n = len(existing_nodes) - new_n = G.number_of_nodes() - if new_n < existing_n: - raise ValueError( - f"graphify: build_merge would shrink graph from {existing_n} → {new_n} nodes. " - f"Pass prune_sources explicitly if you intend to remove nodes." - ) -``` - -- [ ] **Step 5: Run tests** - -```bash -venv/bin/python -m pytest tests/test_dedup.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all pass including `test_build_calls_dedup`. - -- [ ] **Step 6: Run full suite** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: 532 passed, 5 failed (same pre-existing SQL failures). - -- [ ] **Step 7: Commit** - -```bash -git add graphify/build.py -git commit -m "Wire deduplicate_entities into build() and build_merge()" -``` - ---- - -## Task 4: Incremental updates — semantic cache + manifest in `__main__.py` - -**Files:** -- Modify: `graphify/__main__.py` (lines 1971–2117, the extract block) -- Create: `tests/test_incremental.py` - -- [ ] **Step 1: Write failing tests** - -Create `tests/test_incremental.py`: - -```python -"""Integration tests for incremental graphify extract behavior.""" -from __future__ import annotations -import json -import subprocess -import sys -from pathlib import Path - -import pytest - -PYTHON = sys.executable -FIXTURES = Path(__file__).parent / "fixtures" - - -def _run(args: list[str], cwd: Path) -> subprocess.CompletedProcess: - return subprocess.run( - [PYTHON, "-m", "graphify"] + args, - cwd=cwd, - capture_output=True, - text=True, - ) - - -def _make_docs_corpus(tmp_path: Path) -> Path: - """Create a minimal docs corpus with manifest + graph.json for incremental testing.""" - docs = tmp_path / "docs" - docs.mkdir() - (docs / "intro.md").write_text("# Introduction\nThis doc introduces the system.") - (docs / "api.md").write_text("# API Reference\nThe API has endpoints.") - return docs - - -def test_manifest_written_after_extract(tmp_path): - """After a full extract run, manifest.json must exist.""" - # We can't do a real LLM extract in CI, but we can test that the manifest - # path is resolved correctly by checking that a missing API key exits early - # before writing the manifest — and that the path would be correct. - docs = _make_docs_corpus(tmp_path) - r = _run(["extract", str(docs)], tmp_path) - # Should fail with no API key — but NOT with a path error - assert "no LLM API key" in r.stderr or r.returncode != 0 - # manifest should NOT exist (run failed before writing) - manifest = docs / "graphify-out" / "manifest.json" - assert not manifest.exists() - - -def test_incremental_mode_detected_via_manifest(tmp_path): - """If manifest.json + graph.json exist, incremental mode message is shown.""" - docs = _make_docs_corpus(tmp_path) - out = docs / "graphify-out" - out.mkdir() - # Fake a prior successful run - (out / "graph.json").write_text(json.dumps({"nodes": [], "links": []})) - (out / "manifest.json").write_text(json.dumps({"document": [str(docs / "intro.md")]})) - r = _run(["extract", str(docs)], tmp_path) - # Should show incremental scan message (even if it fails on API key) - assert "incremental" in r.stderr.lower() or "incremental" in r.stdout.lower() or r.returncode != 0 - - -def test_no_incremental_without_manifest(tmp_path): - """Without manifest.json, full scan message is shown.""" - docs = _make_docs_corpus(tmp_path) - r = _run(["extract", str(docs)], tmp_path) - # Full scan message (not incremental), then fails on API key - assert "incremental" not in r.stdout -``` - -- [ ] **Step 2: Run — verify tests fail or pass trivially** - -```bash -venv/bin/python -m pytest tests/test_incremental.py -v --tb=short 2>&1 | tail -20 -``` - -Note current behavior before changes. - -- [ ] **Step 3: Update the `elif cmd == "extract":` block in `__main__.py`** - -Find line 1971 (the `from graphify.detect import detect as _detect` line). Replace the detect + file-list block (lines 1971–1984) with: - -```python - from graphify.detect import ( - detect as _detect, - detect_incremental as _detect_incremental, - save_manifest as _save_manifest, - ) - manifest_path = graphify_out / "manifest.json" - existing_graph_path = graphify_out / "graph.json" - incremental_mode = manifest_path.exists() and existing_graph_path.exists() - - if incremental_mode: - print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental(target, manifest_path=str(manifest_path)) - else: - print(f"[graphify extract] scanning {target}") - detection = _detect(target) - - files_by_type = detection.get("files", {}) - if incremental_mode: - new_by_type = detection.get("new_files", {}) - code_files = [Path(p) for p in new_by_type.get("code", [])] - doc_files = [Path(p) for p in new_by_type.get("document", [])] - paper_files = [Path(p) for p in new_by_type.get("paper", [])] - image_files = [Path(p) for p in new_by_type.get("image", [])] - deleted_files = list(detection.get("deleted_files", [])) - unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) - else: - code_files = [Path(p) for p in files_by_type.get("code", [])] - doc_files = [Path(p) for p in files_by_type.get("document", [])] - paper_files = [Path(p) for p in files_by_type.get("paper", [])] - image_files = [Path(p) for p in files_by_type.get("image", [])] - deleted_files = [] - unchanged_total = 0 - - semantic_files = doc_files + paper_files + image_files - if incremental_mode: - print( - f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " - f"{len(paper_files)} papers, {len(image_files)} images changed; " - f"{unchanged_total} unchanged; {len(deleted_files)} deleted" - ) - else: - print( - f"[graphify extract] found {len(code_files)} code, " - f"{len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images" - ) -``` - -- [ ] **Step 4: Wrap the semantic LLM call with semantic cache** - -Find the semantic extraction block (lines 1998–2024). Replace with: - -```python - from graphify.cache import ( - check_semantic_cache as _check_semantic_cache, - save_semantic_cache as _save_semantic_cache, - ) - sem_result: dict = { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, - } - sem_cache_hits = 0 - sem_cache_misses = 0 - if semantic_files: - sem_paths_str = [str(p) for p in semantic_files] - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=target) - ) - sem_cache_hits = len(semantic_files) - len(uncached_paths) - sem_cache_misses = len(uncached_paths) - sem_result["nodes"].extend(cached_nodes) - sem_result["edges"].extend(cached_edges) - sem_result["hyperedges"].extend(cached_hyperedges) - if sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") - - if uncached_paths: - print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") - try: - fresh = _extract_corpus_parallel( - [Path(p) for p in uncached_paths], - backend=backend, - root=target, - ) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - except Exception as exc: - print(f"[graphify extract] semantic extraction failed: {exc}", file=sys.stderr) - fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - try: - _save_semantic_cache( - fresh.get("nodes", []), - fresh.get("edges", []), - fresh.get("hyperedges", []), - root=target, - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - sem_result["nodes"].extend(fresh.get("nodes", [])) - sem_result["edges"].extend(fresh.get("edges", [])) - sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) - sem_result["input_tokens"] += fresh.get("input_tokens", 0) - sem_result["output_tokens"] += fresh.get("output_tokens", 0) -``` - -- [ ] **Step 5: Replace `build_from_json` with `build_merge` in incremental mode** - -Find the build block starting around line 2063. Replace: - -```python - from graphify.build import build_from_json as _build_from_json - ... - G = _build_from_json(merged) -``` - -With: - -```python - from graphify.build import ( - build_from_json as _build_from_json, - build_merge as _build_merge, - ) - from graphify.cluster import cluster as _cluster, score_all as _score_all - from graphify.export import to_json as _to_json - from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - - if incremental_mode: - G = _build_merge( - [merged], - graph_path=graph_json_path, - prune_sources=deleted_files or None, - dedup=True, - ) - else: - G = _build_from_json(merged) -``` - -- [ ] **Step 6: Add manifest save after successful write** - -Find the `analysis_path.write_text(...)` line (around line 2101). After it, add: - -```python - try: - _save_manifest( - detection.get("files", {}), - manifest_path=str(manifest_path), - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) -``` - -Also add the same block in the `--no-cluster` path after `graph_json_path.write_text(...)` (around line 2043). - -- [ ] **Step 7: Add `--dedup-llm` flag parsing** - -In the args-parsing while loop (around lines 1913–1928), add: - -```python - elif a == "--dedup-llm": - dedup_llm = True; i += 1 -``` - -And initialize `dedup_llm = False` before the loop. - -- [ ] **Step 8: Update summary print at end** - -Replace the final summary lines (around 2104–2116) with: - -```python - print( - f"[graphify extract] wrote {graph_json_path}: " - f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " - f"{len(communities)} communities" - ) - print(f"[graphify extract] wrote {analysis_path}") - if incremental_mode: - print( - f"[graphify extract] incremental summary: " - f"{sem_cache_hits + unchanged_total} files cached/unchanged, " - f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted" - ) - elif sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost (~{backend}): ${cost:.4f}" - ) -``` - -- [ ] **Step 9: Run incremental tests** - -```bash -venv/bin/python -m pytest tests/test_incremental.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all pass. - -- [ ] **Step 10: Run full suite** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: 532+ passed, 5 failed (pre-existing SQL). - -- [ ] **Step 11: Commit** - -```bash -git add graphify/__main__.py tests/test_incremental.py -git commit -m "Add incremental updates to graphify extract: semantic cache + build_merge + manifest" -``` - ---- - -## Task 5: Add `--dedup-llm` tiebreaker to `dedup.py` - -**Files:** -- Modify: `graphify/dedup.py` - -- [ ] **Step 1: Write failing test** - -Add to `tests/test_dedup.py`: - -```python -def test_dedup_llm_flag_accepted(): - """deduplicate_entities accepts dedup_llm_backend without crashing when no ambiguous pairs exist.""" - nodes = _make_nodes("UserService", "OrderService") - edges = [] - # Should not crash even with dedup_llm_backend set — just nothing to resolve - result_nodes, _ = deduplicate_entities(nodes, edges, communities={}, dedup_llm_backend=None) - assert len(result_nodes) == 2 -``` - -- [ ] **Step 2: Run — verify it fails** - -```bash -venv/bin/python -m pytest tests/test_dedup.py::test_dedup_llm_flag_accepted -v --tb=short -``` - -Expected: FAIL — `deduplicate_entities` does not accept `dedup_llm_backend` kwarg. - -- [ ] **Step 3: Add `dedup_llm_backend` param to `deduplicate_entities`** - -Update the signature in `graphify/dedup.py`: - -```python -def deduplicate_entities( - nodes: list[dict], - edges: list[dict], - *, - communities: dict[str, int], - dedup_llm_backend: str | None = None, -) -> tuple[list[dict], list[dict]]: -``` - -After the fuzzy merge loop (before building the remap table), add the LLM tiebreaker block: - -```python - # ── pass 3: LLM tiebreaker for ambiguous pairs (opt-in) ────────────────── - if dedup_llm_backend is not None: - _llm_tiebreak(candidates, uf, communities, backend=dedup_llm_backend) -``` - -Add the helper at the bottom of `dedup.py`: - -```python -def _llm_tiebreak( - candidates: list[dict], - uf: _UF, - communities: dict[str, int], - *, - backend: str, - batch_size: int = 30, - low: float = 75.0, - high: float = 92.0, -) -> None: - """Batch-resolve ambiguous pairs (score in [low, high)) via LLM.""" - try: - from graphify.llm import extract_corpus_parallel as _llm # noqa: F401 - import os - from graphify.llm import BACKENDS - env_key = BACKENDS.get(backend, {}).get("env_key", "") - if not os.environ.get(env_key): - print(f"[graphify] --dedup-llm: {env_key} not set, skipping LLM tiebreaker.", flush=True) - return - except ImportError: - return - - # Collect ambiguous pairs - ambiguous: list[tuple[dict, dict, float]] = [] - for i, node in enumerate(candidates): - norm_i = _norm(node.get("label", node.get("id", ""))) - for j in range(i + 1, len(candidates)): - neighbor = candidates[j] - if uf.find(node["id"]) == uf.find(neighbor["id"]): - continue - norm_j = _norm(neighbor.get("label", neighbor.get("id", ""))) - score = fuzz.jaro_winkler_similarity(norm_i, norm_j) * 100 - c1 = communities.get(node["id"]) - c2 = communities.get(neighbor["id"]) - if c1 is not None and c2 is not None and c1 == c2: - score += _COMMUNITY_BOOST - if low <= score < high: - ambiguous.append((node, neighbor, score)) - - if not ambiguous: - return - - # Batch into groups of batch_size and call LLM - try: - from graphify.llm import _call_llm - except ImportError: - return - - for batch_start in range(0, len(ambiguous), batch_size): - batch = ambiguous[batch_start : batch_start + batch_size] - pairs_text = "\n".join( - f"{i+1}. \"{a['label']}\" vs \"{b['label']}\"" - for i, (a, b, _) in enumerate(batch) - ) - prompt = ( - "For each pair below, answer only 'yes' or 'no': are they the same real-world concept?\n\n" - f"{pairs_text}\n\n" - "Reply with one line per pair: '1. yes', '2. no', etc." - ) - try: - response = _call_llm(prompt, backend=backend, max_tokens=200) - lines = response.strip().splitlines() - for line in lines: - line = line.strip() - if not line: - continue - parts = line.split(".", 1) - if len(parts) != 2: - continue - try: - idx = int(parts[0].strip()) - 1 - except ValueError: - continue - if 0 <= idx < len(batch): - answer = parts[1].strip().lower() - if answer.startswith("yes"): - a, b, _ = batch[idx] - winner = _pick_winner([a, b]) - uf.union(winner["id"], a["id"]) - uf.union(winner["id"], b["id"]) - except Exception as exc: - print(f"[graphify] --dedup-llm batch failed: {exc}", flush=True) -``` - -- [ ] **Step 4: Run tests** - -```bash -venv/bin/python -m pytest tests/test_dedup.py -v --tb=short 2>&1 | tail -20 -``` - -Expected: all pass. - -- [ ] **Step 5: Run full suite** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: 532+ passed, 5 failed. - -- [ ] **Step 6: Commit** - -```bash -git add graphify/dedup.py tests/test_dedup.py -git commit -m "Add --dedup-llm LLM tiebreaker to dedup pipeline" -``` - ---- - -## Task 6: Update CHANGELOG + bump version to 0.7.5 - -**Files:** -- Modify: `CHANGELOG.md` -- Modify: `pyproject.toml` - -- [ ] **Step 1: Add changelog entry** - -Add at the top of `CHANGELOG.md`: - -```markdown -## 0.7.5 (2026-05-04) - -- Feat: `graphify extract` now runs incrementally — auto-detects prior `manifest.json` and re-extracts only changed/new files; semantic results cached by content hash so unchanged docs cost zero LLM tokens on repeat runs (#698) -- Feat: Entity deduplication pipeline runs on every build — entropy gate + MinHash/LSH blocking + Jaro-Winkler verification + same-community boost collapses near-duplicate entities (typos, spacing, plurals) before clustering -- Feat: `--dedup-llm` flag for `graphify extract` — optional LLM tiebreaker for ambiguous entity pairs (~$0.01 for 10k-node graphs), off by default -- Deps: `datasketch` and `rapidfuzz` added as base dependencies -``` - -- [ ] **Step 2: Bump version** - -In `pyproject.toml`, change: -```toml -version = "0.7.4" -``` -to: -```toml -version = "0.7.5" -``` - -- [ ] **Step 3: Run full suite one final time** - -```bash -venv/bin/python -m pytest tests/ -q --tb=no 2>&1 | tail -5 -``` - -Expected: 532+ passed, 5 failed (pre-existing SQL only). - -- [ ] **Step 4: Commit** - -```bash -git add CHANGELOG.md pyproject.toml -git commit -m "bump version to 0.7.5" -``` - ---- - -## Self-Review - -**Spec coverage:** -- [x] Semantic cache wrapping → Task 4 steps 3-4 -- [x] Incremental auto-detection via manifest → Task 4 step 3 -- [x] `build_merge` with `prune_sources` → Task 4 step 5 -- [x] Manifest saved on success only → Task 4 step 6 -- [x] Summary print → Task 4 step 8 -- [x] `dedup.py` new module → Task 2 -- [x] Entropy gate → Task 2 step 3 -- [x] MinHash/LSH blocking → Task 2 step 3 -- [x] Jaro-Winkler verification → Task 2 step 3 -- [x] Same-community boost → Task 2 step 3 -- [x] Union-find merge → Task 2 step 3 -- [x] `--dedup-llm` tiebreaker → Task 5 -- [x] Wire dedup into `build()` and `build_merge()` → Task 3 -- [x] `datasketch` + `rapidfuzz` deps → Task 1 -- [x] Tests for all dedup steps → Task 2 + 3 -- [x] Tests for incremental → Task 4 -- [x] CHANGELOG + version bump → Task 6 - -**Placeholder scan:** None found. - -**Type consistency:** `deduplicate_entities(nodes, edges, *, communities, dedup_llm_backend=None)` used consistently in Task 2, Task 3, Task 5. `build(extractions, *, directed, dedup)` consistent in Task 3. `build_merge(..., dedup=True)` consistent in Task 3 and Task 4. diff --git a/docs/superpowers/specs/2026-05-04-incremental-updates-dedup-design.md b/docs/superpowers/specs/2026-05-04-incremental-updates-dedup-design.md deleted file mode 100644 index 217e24d65..000000000 --- a/docs/superpowers/specs/2026-05-04-incremental-updates-dedup-design.md +++ /dev/null @@ -1,133 +0,0 @@ -# Design: Incremental Updates + Entity Deduplication - -**Date:** 2026-05-04 -**Issues:** #698 (incremental updates), entity deduplication (no issue, proactive) -**Branch:** v7 - ---- - -## Problem - -1. `graphify extract` rebuilds the full graph from scratch every run — re-sends all files to the LLM regardless of what changed. For a 1000-file Markdown corpus updated daily this is expensive. - -2. LLM extraction is chunk-by-chunk — the same real-world concept can get different labels across chunks (`AuthManager`, `AuthenticationManager`, `auth_mgr`). No semantic dedup exists beyond exact string normalization. - ---- - -## Pipeline (every `graphify extract` run) - -``` -detect (full or incremental, auto-detected) - ↓ -AST extract (code files, AST cache-aware) - ↓ -Semantic LLM extract (doc/paper/image files, semantic cache-aware) - ↓ -build_merge (merge into existing graph, prune deleted nodes) - ↓ -deduplicate_entities (normalize → entropy gate → MinHash/LSH → Jaro-Winkler → community boost → optional LLM) - ↓ -cluster (full graph, always re-run) - ↓ -score_all + god_nodes + surprising_connections - ↓ -write graph.json + .graphify_analysis.json + manifest.json -``` - ---- - -## Feature 1: Incremental Updates - -### Auto-detection -If `graphify-out/manifest.json` + `graphify-out/graph.json` both exist → incremental mode. No flag needed. First run is always full. - -### Incremental mode changes -- `detect_incremental(target)` instead of `detect(target)` — returns `new_files`, `unchanged_files`, `deleted_files` -- Only `new_files` go through AST + LLM extraction -- `build_merge(new_chunks, prune_sources=deleted_files)` instead of `build_from_json` — merges into existing graph, prunes nodes from deleted files -- `manifest.json` written only on successful completion (crash mid-run does not corrupt next run's diff) - -### Semantic cache (both full and incremental mode) -- Before LLM call: `check_semantic_cache(files)` splits into `(cached_results, uncached_files)` -- Only `uncached_files` sent to `extract_corpus_parallel` -- After LLM call: `save_semantic_cache(fresh_results)` — keyed by content hash -- On file rename: `source_file` updated to new path in cached result (same pattern as existing AST cache) - -### Output summary -``` -[graphify extract] incremental: 20 changed, 980 cached, 2 deleted -[graphify extract] graph: 4,821 nodes, 12,304 edges, 43 communities -[graphify extract] tokens: 18,432 in / 6,201 out, est. cost: $0.08 -``` - -### Files changed -- `graphify/__main__.py` — `elif cmd == "extract":` block only (~5 targeted changes) - ---- - -## Feature 2: Entity Deduplication - -### New module: `graphify/dedup.py` -Single responsibility. Called from `build.py` after graph construction. Returns deduplicated `(nodes, edges)`. - -### Pipeline - -**Step 1 — Exact normalization** -Wire up the dormant `deduplicate_by_label` in `build.py`. Catches case/punctuation variants across files. Free win, already written. - -**Step 2 — Entropy gate** -Skip fuzzy matching on labels with < 2.5 bits/char entropy. Short ambiguous names (`"AI"`, `"DB"`, `"x"`) are too risky to auto-merge. Only high-entropy labels proceed to steps 3-4. - -**Step 3 — MinHash + LSH blocking** (`datasketch`) -3-gram shingles, 128 permutations, threshold 0.7. Generates candidate pairs in O(n) instead of O(n²). Sub-second at 10k nodes. - -**Step 4 — Jaro-Winkler verification** (`rapidfuzz`) -Each candidate pair verified at ≥ 0.92. Catches typos, plurals, spacing variants. Pairs below threshold discarded. - -**Step 5 — Same-community boost** -Pairs where both nodes share a Leiden community ID get +0.05 score bonus. Graphify-specific advantage — community structure is a strong signal that GraphRAG/LightRAG don't exploit. - -**Step 6 — Union-find merge** -Confirmed pairs fed into union-find → connected components → each component merged into one node. Edges rewired to survivor. Self-loops dropped. Prefer shorter non-chunk-suffixed IDs as survivor. - -**Step 7 — Optional LLM tiebreaker** (`--dedup-llm` flag) -Ambiguous pairs (score 0.75–0.85) batched in groups of 30, one LLM call per batch. ~$0.01 total for 10k nodes. Off by default. - -### Integration point -Dedup runs after `build_merge` / `build_from_json`, before `cluster`. Order matters: cleaner graph → better community detection. - -```python -# in build.py -G = build_merge(...) # or build_from_json -G = deduplicate_entities(G) # new step -communities = cluster(G) # unchanged -``` - -### New dependencies -- `datasketch` — always required (added to `[project.dependencies]`) -- `rapidfuzz` — always required (added to `[project.dependencies]`) -- No `sentence-transformers` / PyTorch dependency - -### Files changed -- `graphify/dedup.py` — new module, full pipeline -- `graphify/build.py` — call `deduplicate_entities` after graph construction; wire dormant `deduplicate_by_label` -- `graphify/__main__.py` — add `--dedup-llm` flag parsing in extract block -- `pyproject.toml` — add `datasketch`, `rapidfuzz` to base dependencies - ---- - -## Testing - -- Unit tests for each dedup step in isolation (`tests/test_dedup.py`) -- Integration test: two chunks with overlapping entity labels → single merged node in output graph -- Incremental test: run extract twice, assert second run makes zero LLM calls for unchanged files -- Rename test: rename a file, assert cache hit and `source_file` updated correctly -- Delete test: delete a file, assert its nodes are pruned from graph - ---- - -## Non-goals - -- `--dedup embed` (MiniLM cosine) — explicitly excluded, no PyTorch dependency -- Incremental support for `graphify update` (AST-only) — already handled by existing AST cache -- Dedup across different graph.json files (merge two graphs) — separate feature diff --git a/docs/translations/README.ar-SA.md b/docs/translations/README.ar-SA.md index f3f4a78c2..e6bd35759 100644 --- a/docs/translations/README.ar-SA.md +++ b/docs/translations/README.ar-SA.md @@ -32,7 +32,7 @@ graphify-out/ ├── graph.html رسم بياني تفاعلي — افتحه في أي متصفح، انقر على العقد، ابحث، صفّ ├── GRAPH_REPORT.md عقد الإله، الاتصالات المفاجئة، الأسئلة المقترحة -├── graph.json رسم بياني دائم — استعلم بعد أسابيع دون إعادة القراءة +├── graph.helix رسم بياني دائم — استعلم بعد أسابيع دون إعادة القراءة └── cache/ ذاكرة تخزين مؤقت SHA256 — إعادة التشغيل تعالج الملفات المتغيرة فقط ``` @@ -56,7 +56,7 @@ dist/ ## كيف يعمل -يعمل graphify في ثلاث مراحل. أولاً، تمريرة AST حتمية تستخرج البنية من ملفات الكود (الفئات، الدوال، الاستيرادات، رسوم بيانية الاستدعاء، docstrings، تعليقات المبرر) — دون الحاجة إلى LLM. ثانياً، يتم نسخ ملفات الفيديو والصوت محلياً باستخدام faster-whisper. ثالثاً، تعمل عوامل Claude الفرعية بالتوازي على المستندات والأوراق البحثية والصور والنصوص المكتوبة لاستخراج المفاهيم والعلاقات ومبررات التصميم. يتم دمج النتائج في رسم بياني NetworkX وتجميعها باستخدام Leiden وتصديرها كـ HTML تفاعلي وJSON قابل للاستعلام وتقرير تدقيق بلغة طبيعية. +يعمل graphify في ثلاث مراحل. أولاً، تمريرة AST حتمية تستخرج البنية من ملفات الكود (الفئات، الدوال، الاستيرادات، رسوم بيانية الاستدعاء، docstrings، تعليقات المبرر) — دون الحاجة إلى LLM. ثانياً، يتم نسخ ملفات الفيديو والصوت محلياً باستخدام faster-whisper. ثالثاً، تعمل عوامل Claude الفرعية بالتوازي على المستندات والأوراق البحثية والصور والنصوص المكتوبة لاستخراج المفاهيم والعلاقات ومبررات التصميم. يتم دمج النتائج في رسم بياني Helix وتجميعها باستخدام Leiden وتصديرها كـ HTML تفاعلي وJSON قابل للاستعلام وتقرير تدقيق بلغة طبيعية. **التجميع مبني على طوبولوجيا الرسم البياني — بدون embeddings.** يجد Leiden المجتمعات بواسطة كثافة الحواف. حواف التشابه الدلالي التي يستخرجها Claude (`semantically_similar_to`، مصنفة INFERRED) موجودة بالفعل في الرسم البياني. بنية الرسم البياني هي إشارة التشابه — لا حاجة لخطوة embedding منفصلة أو قاعدة بيانات متجهية. @@ -162,7 +162,7 @@ graphify watch ./src # تحديث تلقائي للرسم البي ## المكدس التقني -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. استخراج دلالي عبر Claude أو GPT-4 أو نموذج منصتك. نسخ الفيديو عبر faster-whisper + yt-dlp (اختياري). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. استخراج دلالي عبر Claude أو GPT-4 أو نموذج منصتك. نسخ الفيديو عبر faster-whisper + yt-dlp (اختياري). ## مبني على graphify — Penpax diff --git a/docs/translations/README.cs-CZ.md b/docs/translations/README.cs-CZ.md index db4cb2577..7150b6fe2 100644 --- a/docs/translations/README.cs-CZ.md +++ b/docs/translations/README.cs-CZ.md @@ -27,13 +27,13 @@ Plně multimodální. Přidejte kód, PDF, markdown, snímky obrazovky, diagramy graphify-out/ ├── graph.html interaktivní graf — otevřete v libovolném prohlížeči ├── GRAPH_REPORT.md boží uzly, překvapivá propojení, navrhované otázky -├── graph.json trvalý graf — dotazovatelný týdny poté +├── graph.helix trvalý graf — dotazovatelný týdny poté └── cache/ SHA256 cache — opakovaná spuštění zpracovávají pouze změněné soubory ``` ## Jak to funguje -graphify pracuje ve třech průchodech. Nejprve deterministický průchod AST extrahuje strukturu z kódových souborů bez LLM. Poté jsou video a zvukové soubory přepisovány lokálně pomocí faster-whisper. Nakonec sub-agenti Claude běží paralelně na dokumentech, článcích, obrázcích a přepisech. Výsledky jsou sloučeny do grafu NetworkX, clusterovány pomocí Leiden a exportovány jako interaktivní HTML, dotazovatelný JSON a auditní zpráva. +graphify pracuje ve třech průchodech. Nejprve deterministický průchod AST extrahuje strukturu z kódových souborů bez LLM. Poté jsou video a zvukové soubory přepisovány lokálně pomocí faster-whisper. Nakonec sub-agenti Claude běží paralelně na dokumentech, článcích, obrázcích a přepisech. Výsledky jsou sloučeny do grafu Helix, clusterovány pomocí Leiden a exportovány jako interaktivní HTML, dotazovatelný JSON a auditní zpráva. Každý vztah je označen `EXTRACTED`, `INFERRED` (se skóre spolehlivosti) nebo `AMBIGUOUS`. diff --git a/docs/translations/README.da-DK.md b/docs/translations/README.da-DK.md index b6ca65e6c..f022437a5 100644 --- a/docs/translations/README.da-DK.md +++ b/docs/translations/README.da-DK.md @@ -27,13 +27,13 @@ Fuldt multimodal. Tilføj kode, PDF'er, markdown, skærmbilleder, diagrammer, wh graphify-out/ ├── graph.html interaktiv graf — åbn i enhver browser ├── GRAPH_REPORT.md gudknuder, overraskende forbindelser, foreslåede spørgsmål -├── graph.json vedvarende graf — forespørgselsbar uger senere +├── graph.helix vedvarende graf — forespørgselsbar uger senere └── cache/ SHA256-cache — gentagne kørsler behandler kun ændrede filer ``` ## Sådan fungerer det -graphify arbejder i tre gennemløb. Først udtrækker et deterministisk AST-gennemløb struktur fra kodefiler uden LLM. Derefter transskriberes video- og lydfiler lokalt med faster-whisper. Endelig kører Claude-underagenter parallelt på dokumenter, artikler, billeder og transskriptioner. Resultaterne flettes ind i en NetworkX-graf, klynges med Leiden og eksporteres som interaktiv HTML, forespørgselsbar JSON og revisionsrapport. +graphify arbejder i tre gennemløb. Først udtrækker et deterministisk AST-gennemløb struktur fra kodefiler uden LLM. Derefter transskriberes video- og lydfiler lokalt med faster-whisper. Endelig kører Claude-underagenter parallelt på dokumenter, artikler, billeder og transskriptioner. Resultaterne flettes ind i en Helix-graf, klynges med Leiden og eksporteres som interaktiv HTML, forespørgselsbar JSON og revisionsrapport. Hver relation er mærket `EXTRACTED`, `INFERRED` (med konfidensscore) eller `AMBIGUOUS`. diff --git a/docs/translations/README.de-DE.md b/docs/translations/README.de-DE.md index f81067902..268cd0449 100644 --- a/docs/translations/README.de-DE.md +++ b/docs/translations/README.de-DE.md @@ -28,7 +28,7 @@ Vollständig multimodal. Leg Code, PDFs, Markdown, Screenshots, Diagramme, White graphify-out/ ├── graph.html interaktiver Graph — im Browser öffnen, Knoten anklicken, suchen, filtern ├── GRAPH_REPORT.md Gott-Knoten, überraschende Verbindungen, vorgeschlagene Fragen -├── graph.json persistenter Graph — Wochen später abfragen, ohne neu zu lesen +├── graph.helix persistenter Graph — Wochen später abfragen, ohne neu zu lesen └── cache/ SHA256-Cache — erneute Ausführungen verarbeiten nur geänderte Dateien ``` @@ -46,7 +46,7 @@ Gleiche Syntax wie `.gitignore`. Du kannst eine einzelne `.graphifyignore` im Re ## So funktioniert es -graphify läuft in drei Durchgängen. Zuerst extrahiert ein deterministischer AST-Durchgang Strukturen aus Code-Dateien (Klassen, Funktionen, Importe, Aufrufgraphen, Docstrings, Begründungskommentare) — ohne LLM. Zweitens werden Video- und Audiodateien lokal mit faster-whisper transkribiert, angetrieben durch einen domänenspezifischen Prompt aus Korpus-Gott-Knoten — Transkripte werden gecacht, sodass erneute Ausführungen sofort sind. Drittens laufen Claude-Subagenten parallel über Dokumente, Papers, Bilder und Transkripte, um Konzepte, Beziehungen und Designbegründungen zu extrahieren. Die Ergebnisse werden in einem NetworkX-Graphen zusammengeführt, mit Leiden-Community-Erkennung geclustert und als interaktives HTML, abfragbares JSON und ein Klartext-Audit-Report exportiert. +graphify läuft in drei Durchgängen. Zuerst extrahiert ein deterministischer AST-Durchgang Strukturen aus Code-Dateien (Klassen, Funktionen, Importe, Aufrufgraphen, Docstrings, Begründungskommentare) — ohne LLM. Zweitens werden Video- und Audiodateien lokal mit faster-whisper transkribiert, angetrieben durch einen domänenspezifischen Prompt aus Korpus-Gott-Knoten — Transkripte werden gecacht, sodass erneute Ausführungen sofort sind. Drittens laufen Claude-Subagenten parallel über Dokumente, Papers, Bilder und Transkripte, um Konzepte, Beziehungen und Designbegründungen zu extrahieren. Die Ergebnisse werden in einem Helix-Graphen zusammengeführt, mit Leiden-Community-Erkennung geclustert und als interaktives HTML, abfragbares JSON und ein Klartext-Audit-Report exportiert. **Clustering basiert auf Graph-Topologie — keine Embeddings.** Leiden findet Communities durch Kantendichte. Die semantischen Ähnlichkeitskanten, die Claude extrahiert (`semantically_similar_to`, markiert als INFERRED), sind bereits im Graphen, sodass sie die Community-Erkennung direkt beeinflussen. Die Graphstruktur ist das Ähnlichkeitssignal — kein separater Embedding-Schritt oder Vektordatenbank nötig. @@ -167,7 +167,7 @@ graphify sendet Dateiinhalte an die Modell-API deines KI-Assistenten für semant ## Tech-Stack -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantische Extraktion via Claude, GPT-4 oder welches Modell deine Plattform verwendet. Video-Transkription via faster-whisper + yt-dlp (optional). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Semantische Extraktion via Claude, GPT-4 oder welches Modell deine Plattform verwendet. Video-Transkription via faster-whisper + yt-dlp (optional). ## Auf graphify aufgebaut — Penpax diff --git a/docs/translations/README.el-GR.md b/docs/translations/README.el-GR.md index 599e615a1..0c1fa857d 100644 --- a/docs/translations/README.el-GR.md +++ b/docs/translations/README.el-GR.md @@ -27,13 +27,13 @@ graphify-out/ ├── graph.html διαδραστικός γράφος — ανοίξτε σε οποιοδήποτε πρόγραμμα περιήγησης ├── GRAPH_REPORT.md κόμβοι-θεοί, εκπληκτικές συνδέσεις, προτεινόμενες ερωτήσεις -├── graph.json επίμονος γράφος — μπορεί να υποβληθεί σε ερωτήματα εβδομάδες αργότερα +├── graph.helix επίμονος γράφος — μπορεί να υποβληθεί σε ερωτήματα εβδομάδες αργότερα └── cache/ κρυφή μνήμη SHA256 — επαναλαμβανόμενες εκτελέσεις επεξεργάζονται μόνο τα αλλαγμένα αρχεία ``` ## Πώς λειτουργεί -Το graphify λειτουργεί σε τρεις διελεύσεις. Πρώτα, μια ντετερμινιστική διέλευση AST εξάγει δομή από αρχεία κώδικα χωρίς LLM. Στη συνέχεια, τα αρχεία βίντεο και ήχου μεταγράφονται τοπικά με faster-whisper. Τέλος, οι υπο-πράκτορες Claude εκτελούνται παράλληλα σε έγγραφα, εργασίες, εικόνες και μεταγραφές. Τα αποτελέσματα συγχωνεύονται σε ένα γράφο NetworkX, ομαδοποιούνται με Leiden και εξάγονται ως διαδραστική HTML, JSON για ερωτήματα και αναφορά ελέγχου. +Το graphify λειτουργεί σε τρεις διελεύσεις. Πρώτα, μια ντετερμινιστική διέλευση AST εξάγει δομή από αρχεία κώδικα χωρίς LLM. Στη συνέχεια, τα αρχεία βίντεο και ήχου μεταγράφονται τοπικά με faster-whisper. Τέλος, οι υπο-πράκτορες Claude εκτελούνται παράλληλα σε έγγραφα, εργασίες, εικόνες και μεταγραφές. Τα αποτελέσματα συγχωνεύονται σε ένα γράφο Helix, ομαδοποιούνται με Leiden και εξάγονται ως διαδραστική HTML, JSON για ερωτήματα και αναφορά ελέγχου. Κάθε σχέση επισημαίνεται ως `EXTRACTED`, `INFERRED` (με βαθμολογία εμπιστοσύνης) ή `AMBIGUOUS`. diff --git a/docs/translations/README.es-ES.md b/docs/translations/README.es-ES.md index 7c58a2b27..bd73f7644 100644 --- a/docs/translations/README.es-ES.md +++ b/docs/translations/README.es-ES.md @@ -28,7 +28,7 @@ Totalmente multimodal. Deposita código, PDFs, markdown, capturas de pantalla, d graphify-out/ ├── graph.html grafo interactivo — abrir en cualquier navegador, hacer clic en nodos, buscar ├── GRAPH_REPORT.md nodos dios, conexiones sorprendentes, preguntas sugeridas -├── graph.json grafo persistente — consultar semanas después sin releer +├── graph.helix grafo persistente — consultar semanas después sin releer └── cache/ caché SHA256 — las re-ejecuciones solo procesan archivos modificados ``` @@ -46,7 +46,7 @@ Misma sintaxis que `.gitignore`. Puedes mantener un único `.graphifyignore` en ## Cómo funciona -graphify se ejecuta en tres pasadas. Primero, una pasada AST determinista extrae estructura de los archivos de código (clases, funciones, importaciones, grafos de llamadas, docstrings, comentarios de justificación) sin necesidad de LLM. Segundo, los archivos de video y audio se transcriben localmente con faster-whisper usando un prompt adaptado al dominio derivado de los nodos dios del corpus. Tercero, subagentes de Claude se ejecutan en paralelo sobre documentos, papers, imágenes y transcripciones para extraer conceptos, relaciones y justificaciones de diseño. Los resultados se fusionan en un grafo NetworkX, se agrupan con detección de comunidades Leiden, y se exportan como HTML interactivo, JSON consultable y un informe de auditoría en lenguaje natural. +graphify se ejecuta en tres pasadas. Primero, una pasada AST determinista extrae estructura de los archivos de código (clases, funciones, importaciones, grafos de llamadas, docstrings, comentarios de justificación) sin necesidad de LLM. Segundo, los archivos de video y audio se transcriben localmente con faster-whisper usando un prompt adaptado al dominio derivado de los nodos dios del corpus. Tercero, subagentes de Claude se ejecutan en paralelo sobre documentos, papers, imágenes y transcripciones para extraer conceptos, relaciones y justificaciones de diseño. Los resultados se fusionan en un grafo Helix, se agrupan con detección de comunidades Leiden, y se exportan como HTML interactivo, JSON consultable y un informe de auditoría en lenguaje natural. **El clustering se basa en la topología del grafo — sin embeddings.** Leiden encuentra comunidades por densidad de aristas. Las aristas de similitud semántica que Claude extrae (`semantically_similar_to`, marcadas como INFERRED) ya están en el grafo. La estructura del grafo es la señal de similitud — no se necesita paso de embedding separado ni base de datos vectorial. @@ -157,7 +157,7 @@ graphify envía contenido de archivos a la API del modelo de tu asistente IA par ## Stack técnico -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Extracción semántica via Claude, GPT-4 o el modelo de tu plataforma. Transcripción de video via faster-whisper + yt-dlp (opcional). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Extracción semántica via Claude, GPT-4 o el modelo de tu plataforma. Transcripción de video via faster-whisper + yt-dlp (opcional). ## Construido sobre graphify — Penpax diff --git a/docs/translations/README.fa-IR.md b/docs/translations/README.fa-IR.md index 2851c0803..ae0819440 100644 --- a/docs/translations/README.fa-IR.md +++ b/docs/translations/README.fa-IR.md @@ -40,7 +40,7 @@ graphify-out/ ├── graph.html در هر مرورگری باز کنید — روی گره‌ها کلیک کنید، فیلتر کنید، جستجو کنید ├── GRAPH_REPORT.md نکات کلیدی: مفاهیم محوری، اتصالات شگفت‌انگیز، سؤالات پیشنهادی -└── graph.json گراف کامل — هر زمان بدون نیاز به بازخوانی فایل‌ها پرس‌وجو کنید +└── graph.helix گراف کامل — هر زمان بدون نیاز به بازخوانی فایل‌ها پرس‌وجو کنید ```
@@ -355,14 +355,14 @@ graphify-out/cost.json # فقط محلی ```bash # پرس‌وجو از گراف در ترمینال graphify query "جریان احراز هویت را نشان بده" -graphify query "چه چیزی DigestAuth را به Response متصل می‌کند؟" --graph graphify-out/graph.json +graphify query "چه چیزی DigestAuth را به Response متصل می‌کند؟" --graph graphify-out/graph.helix # نمایش گراف به‌عنوان سرور MCP -python -m graphify.serve graphify-out/graph.json +python -m graphify.serve graphify-out/graph.helix # یا سرویس‌دهی از طریق HTTP برای دسترسی تیمی: -python -m graphify.serve graphify-out/graph.json --transport http --port 8080 -python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +python -m graphify.serve graphify-out/graph.helix --transport http --port 8080 +python -m graphify.serve graphify-out/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ```
diff --git a/docs/translations/README.fi-FI.md b/docs/translations/README.fi-FI.md index afe725881..8e69d6b22 100644 --- a/docs/translations/README.fi-FI.md +++ b/docs/translations/README.fi-FI.md @@ -27,13 +27,13 @@ Täysin multimodaalinen. Lisää koodia, PDF:iä, markdownia, kuvakaappauksia, k graphify-out/ ├── graph.html interaktiivinen graafi — avaa missä tahansa selaimessa ├── GRAPH_REPORT.md jumalsolmut, yllättävät yhteydet, ehdotetut kysymykset -├── graph.json pysyvä graafi — kyselytavissa viikkojen kuluttua +├── graph.helix pysyvä graafi — kyselytavissa viikkojen kuluttua └── cache/ SHA256-välimuisti — toistuvat ajot käsittelevät vain muuttuneet tiedostot ``` ## Miten se toimii -graphify toimii kolmessa läpiajossa. Ensin deterministinen AST-läpiajo poimii rakenteen kooditiedostoista ilman LLM:ää. Sitten video- ja äänitiedostot litteroidaan paikallisesti faster-whisperillä. Lopuksi Clauden ala-agentit suoritetaan rinnakkain asiakirjoissa, papereissa, kuvissa ja litteroinneissa. Tulokset yhdistetään NetworkX-graafiin, klusteroidaan Leidenillä ja viedään interaktiivisena HTML:nä, kyselytavissa olevana JSON:na ja tarkastusraporttina. +graphify toimii kolmessa läpiajossa. Ensin deterministinen AST-läpiajo poimii rakenteen kooditiedostoista ilman LLM:ää. Sitten video- ja äänitiedostot litteroidaan paikallisesti faster-whisperillä. Lopuksi Clauden ala-agentit suoritetaan rinnakkain asiakirjoissa, papereissa, kuvissa ja litteroinneissa. Tulokset yhdistetään Helix-graafiin, klusteroidaan Leidenillä ja viedään interaktiivisena HTML:nä, kyselytavissa olevana JSON:na ja tarkastusraporttina. Jokainen suhde on merkitty `EXTRACTED`, `INFERRED` (luottamuspisteineen) tai `AMBIGUOUS`. diff --git a/docs/translations/README.fil-PH.md b/docs/translations/README.fil-PH.md index e318eb68e..4c97c1bd4 100644 --- a/docs/translations/README.fil-PH.md +++ b/docs/translations/README.fil-PH.md @@ -27,13 +27,13 @@ Ganap na multimodal. Magdagdag ng code, PDF, markdown, mga screenshot, diagram, graphify-out/ ├── graph.html interactive na graph — buksan sa kahit anong browser ├── GRAPH_REPORT.md mga god node, nakakagulat na koneksyon, mga iminumungkahing tanong -├── graph.json persistent na graph — maaaring i-query kahit pagkalipas ng mga linggo +├── graph.helix persistent na graph — maaaring i-query kahit pagkalipas ng mga linggo └── cache/ SHA256 cache — ang mga pag-uulit ay nagpo-proseso lang ng mga nabagong file ``` ## Paano Gumagana -Gumagana ang graphify sa tatlong pass. Una, isang deterministikong AST pass ang nag-e-extract ng istruktura mula sa mga code file nang walang LLM. Pagkatapos, ang mga video at audio file ay tina-transcribe nang lokal gamit ang faster-whisper. Panghuli, mga Claude sub-agent ang tumatakbo nang magkakasabay sa mga dokumento, papel, imahe, at transkripsyon. Ang mga resulta ay pinagsasama sa isang NetworkX graph, naka-cluster gamit ang Leiden, at ine-export bilang interactive na HTML, queryable na JSON, at audit report. +Gumagana ang graphify sa tatlong pass. Una, isang deterministikong AST pass ang nag-e-extract ng istruktura mula sa mga code file nang walang LLM. Pagkatapos, ang mga video at audio file ay tina-transcribe nang lokal gamit ang faster-whisper. Panghuli, mga Claude sub-agent ang tumatakbo nang magkakasabay sa mga dokumento, papel, imahe, at transkripsyon. Ang mga resulta ay pinagsasama sa isang Helix graph, naka-cluster gamit ang Leiden, at ine-export bilang interactive na HTML, queryable na JSON, at audit report. Ang bawat relasyon ay may label na `EXTRACTED`, `INFERRED` (may confidence score), o `AMBIGUOUS`. diff --git a/docs/translations/README.fr-FR.md b/docs/translations/README.fr-FR.md index eb57cd638..5ae61144b 100644 --- a/docs/translations/README.fr-FR.md +++ b/docs/translations/README.fr-FR.md @@ -28,7 +28,7 @@ Entièrement multimodal. Déposez du code, des PDFs, du markdown, des captures d graphify-out/ ├── graph.html graphe interactif — ouvrir dans un navigateur, cliquer, rechercher, filtrer ├── GRAPH_REPORT.md nœuds dieu, connexions surprenantes, questions suggérées -├── graph.json graphe persistant — interrogeable des semaines plus tard sans relire +├── graph.helix graphe persistant — interrogeable des semaines plus tard sans relire └── cache/ cache SHA256 — les réexécutions ne traitent que les fichiers modifiés ``` @@ -46,7 +46,7 @@ Même syntaxe que `.gitignore`. Un seul `.graphifyignore` à la racine du dépô ## Comment ça fonctionne -graphify s'exécute en trois passes. D'abord, un passage AST déterministe extrait la structure des fichiers de code (classes, fonctions, imports, graphes d'appel, docstrings, commentaires de justification) sans LLM. Ensuite, les fichiers vidéo et audio sont transcrits localement avec faster-whisper. Enfin, des sous-agents Claude s'exécutent en parallèle sur les docs, articles, images et transcriptions pour extraire concepts, relations et justifications de conception. Les résultats sont fusionnés dans un graphe NetworkX, regroupés avec la détection de communautés Leiden, et exportés en HTML interactif, JSON interrogeable et un rapport d'audit en langage naturel. +graphify s'exécute en trois passes. D'abord, un passage AST déterministe extrait la structure des fichiers de code (classes, fonctions, imports, graphes d'appel, docstrings, commentaires de justification) sans LLM. Ensuite, les fichiers vidéo et audio sont transcrits localement avec faster-whisper. Enfin, des sous-agents Claude s'exécutent en parallèle sur les docs, articles, images et transcriptions pour extraire concepts, relations et justifications de conception. Les résultats sont fusionnés dans un graphe Helix, regroupés avec la détection de communautés Leiden, et exportés en HTML interactif, JSON interrogeable et un rapport d'audit en langage naturel. **Le clustering est basé sur la topologie du graphe — pas d'embeddings.** Leiden trouve les communautés par densité d'arêtes. Les arêtes de similarité sémantique extraites par Claude (`semantically_similar_to`, marquées INFERRED) sont déjà dans le graphe. La structure du graphe est le signal de similarité — pas d'étape d'embedding séparée ni de base de données vectorielle nécessaire. @@ -157,7 +157,7 @@ graphify envoie le contenu des fichiers à l'API du modèle de votre assistant I ## Stack technique -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Extraction sémantique via Claude, GPT-4 ou le modèle de votre plateforme. Transcription vidéo via faster-whisper + yt-dlp (optionnel). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Extraction sémantique via Claude, GPT-4 ou le modèle de votre plateforme. Transcription vidéo via faster-whisper + yt-dlp (optionnel). ## Construit sur graphify — Penpax diff --git a/docs/translations/README.he-IL.md b/docs/translations/README.he-IL.md index 65e9ae75d..8fede1345 100644 --- a/docs/translations/README.he-IL.md +++ b/docs/translations/README.he-IL.md @@ -46,7 +46,7 @@ graphify-out/ ├── graph.html נפתח בכל דפדפן — לחיצה על צמתים, סינון, חיפוש ├── GRAPH_REPORT.md עיקרי הדברים: מושגי מפתח, קשרים מפתיעים, שאלות מוצעות -└── graph.json הגרף המלא — אפשר לשאול עליו בכל רגע בלי לקרוא שוב את הקבצים +└── graph.helix הגרף המלא — אפשר לשאול עליו בכל רגע בלי לקרוא שוב את הקבצים ```
@@ -391,7 +391,7 @@ graphify-out/cost.json # מקומי בלבד **תהליך העבודה:** 1. אחד מחברי הצוות מריץ `‎/graphify .` ומבצע commit ל-`graphify-out/`. 2. כולם מושכים — העוזר שלהם קורא את הגרף מיד. -3. הריצו `graphify hook install` לבנייה אוטומטית מחדש אחרי כל commit ‏(AST בלבד, ללא עלות API). זה גם מגדיר merge driver של git כך ש-`graph.json` לעולם לא יישאר עם סימוני קונפליקט — שני מפתחים שמבצעים commit במקביל מקבלים מיזוג-איחוד אוטומטי של הגרפים. +3. הריצו `graphify hook install` לבנייה אוטומטית מחדש אחרי כל commit ‏(AST בלבד, ללא עלות API). זה גם מגדיר merge driver של git כך ש-`graph.helix` לעולם לא יישאר עם סימוני קונפליקט — שני מפתחים שמבצעים commit במקביל מקבלים מיזוג-איחוד אוטומטי של הגרפים. 4. כשמסמכים או מאמרים משתנים, הריצו `‎/graphify --update` לרענון הצמתים הללו. --- @@ -403,18 +403,18 @@ graphify-out/cost.json # מקומי בלבד ```bash # שאילתת גרף מהטרמינל graphify query "הצג את זרימת האימות" -graphify query "מה מחבר בין DigestAuth ל-Response?" --graph graphify-out/graph.json +graphify query "מה מחבר בין DigestAuth ל-Response?" --graph graphify-out/graph.helix # חשיפת הגרף כשרת MCP (לגישת כלים חוזרת) -python -m graphify.serve graphify-out/graph.json -python -m graphify.serve --graph graphify-out/graph.json # גם הדגל --graph מתקבל +python -m graphify.serve graphify-out/graph.helix +python -m graphify.serve --graph graphify-out/graph.helix # גם הדגל --graph מתקבל # רישום ב-Kimi Code: -kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.json +kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.helix # או הגשה על HTTP כך שכל הצוות מצביע על URL אחד (בלי graphify מקומי): -python -m graphify.serve graphify-out/graph.json --transport http --port 8080 -python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +python -m graphify.serve graphify-out/graph.helix --transport http --port 8080 +python -m graphify.serve graphify-out/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ```
@@ -443,7 +443,7 @@ python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 ```bash docker build -t graphify . docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ - /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" + /data/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET" ```
@@ -495,7 +495,7 @@ python3 -m venv .venv && .venv/bin/pip install "graphifyy[mcp]" | `GRAPHIFY_QUERY_LOG` | דריסת נתיב יומן השאילתות (ברירת מחדל: `~/.cache/graphify-queries.log`) | אופציונלי — ערך ריק או `/dev/null` להשתקה | | `GRAPHIFY_QUERY_LOG_DISABLE` | הגדירו `1` לביטול מוחלט של יומן השאילתות | אופציונלי | | `GRAPHIFY_QUERY_LOG_RESPONSES` | הגדירו `1` לרישום גם של תשובות תת-גרף מלאות (כבוי כברירת מחדל) | אופציונלי | -| `GRAPHIFY_MAX_GRAPH_BYTES` | דריסת תקרת הגודל של graph.json ‏(512 MiB) — למשל `700MB`, ‏`2GB` או בייטים | אופציונלי — שימושי לקורפוסים גדולים מאוד | +| `GRAPHIFY_MAX_GRAPH_BYTES` | דריסת תקרת הגודל של graph.helix ‏(512 MiB) — למשל `700MB`, ‏`2GB` או בייטים | אופציונלי — שימושי לקורפוסים גדולים מאוד | | `GRAPHIFY_LLM_TEMPERATURE` | דריסת טמפרטורת ה-LLM לחילוץ סמנטי — למשל `0.7`, או `none` להשמטה | אופציונלי — מושמט אוטומטית למודלי היסק o1/o3/o4/gpt-5 | --- @@ -579,8 +579,8 @@ graphify query "..."
-**ל-`graph.json` יש סימוני קונפליקט אחרי ששני מפתחים ביצעו commit במקביל** -הריצו `graphify hook install` — הוא מגדיר merge driver של git שממזג-מאחד את `graph.json` אוטומטית כך שקונפליקטים לא קורים בכלל. +**ל-`graph.helix` יש סימוני קונפליקט אחרי ששני מפתחים ביצעו commit במקביל** +הריצו `graphify hook install` — הוא מגדיר merge driver של git שממזג-מאחד את `graph.helix` אוטומטית כך שקונפליקטים לא קורים בכלל. **החילוץ מחזיר צמתים/קשתות ריקים למסמכים או PDF** מסמכים, PDF ותמונות דורשים קריאת LLM — קורפוסים של קוד בלבד אינם דורשים מפתח. ודאו שמפתח ה-API מוגדר וה-backend נכון: @@ -644,7 +644,7 @@ graphify save-result --question "Q" --answer "A" --nodes Foo Bar --outcome usefu graphify reflect # איחוד תוצאות graphify-out/memory/ אל reflections/LESSONS.md graphify reflect --if-stale # לא עושה דבר אם LESSONS.md כבר חדש מכל הקלטים (זול להרצה בכל סשן) graphify reflect --out docs/LESSONS.md # כתיבת מסמך הלקחים למקום אחר -graphify reflect --graph graphify-out/graph.json # קיבוץ לקחים לפי קהילה + כתיבת שכבת זיכרון העבודה (.graphify_learning.json) +graphify reflect --graph graphify-out/graph.helix # קיבוץ לקחים לפי קהילה + כתיבת שכבת זיכרון העבודה (.graphify_learning.json) # השכבה מתייגת צמתים preferred/tentative/contested (משוקלל-עדכניות, עם מקור); # graphify explain / query מציגים אז רמז "Lesson:", מסומן "code changed — re-verify" כשהמקור התקדם @@ -717,7 +717,7 @@ graphify extract ./docs --google-workspace # ייצוא .gdoc/.gsheet/.gslid graphify extract ./docs --mode deep # חילוץ סמנטי עשיר יותר עם system prompt מורחב graphify extract ./docs --no-cluster # חילוץ גולמי בלבד, דילוג על אשכול graphify extract ./docs --timing # הדפסת זמני ריצה פר-שלב ל-stderr ‏(עובד גם ב-cluster-only) -graphify extract ./docs --force # דריסת graph.json גם אם לגרף החדש פחות צמתים (אחרי refactors או לניקוי כפילויות רפאים) +graphify extract ./docs --force # דריסת graph.helix גם אם לגרף החדש פחות צמתים (אחרי refactors או לניקוי כפילויות רפאים) graphify extract ./docs --dedup-llm # ‏LLM כמכריע לזוגות ישויות עמומים (משתמש באותו מפתח API) graphify extract ./docs --global --as myrepo # חילוץ ורישום בגרף הגלובלי חוצה-הפרויקטים GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # הרמת תקרת הפלט לקורפוסים צפופים @@ -727,7 +727,7 @@ graphify export callflow-html --max-sections 8 # הגבלת מספר קטע graphify export callflow-html --output docs/arch.html graphify export callflow-html ./some-repo/graphify-out -graphify global add graphify-out/graph.json --as myrepo # רישום גרף פרויקט אל ~/.graphify/global-graph.json +graphify global add graphify-out/graph.helix --as myrepo # רישום גרף פרויקט אל ~/.graphify/global-graph.helix graphify global remove myrepo # הסרת פרויקט מהגרף הגלובלי graphify global list # הצגת כל המאגרים הרשומים + ספירות צמתים/קשתות graphify global path # הדפסת הנתיב לקובץ הגרף הגלובלי @@ -750,7 +750,7 @@ graphify update ./src graphify update ./src --no-cluster # דילוג על אשכול מחדש, כתיבת גרף AST גולמי בלבד graphify update ./src --force # דריסה גם אם לגרף החדש פחות צמתים graphify cluster-only ./my-project -graphify cluster-only ./my-project --graph path/to/graph.json # מיקום גרף מותאם +graphify cluster-only ./my-project --graph path/to/graph.helix # מיקום גרף מותאם graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # תיוג קהילות מקבילי (גרפים גדולים) graphify cluster-only ./my-project --resolution 1.5 # יותר קהילות, קטנות יותר graphify cluster-only ./my-project --exclude-hubs 99 # החרגת צומתי p99 מהחלוקה diff --git a/docs/translations/README.hi-IN.md b/docs/translations/README.hi-IN.md index 181963c29..b68261174 100644 --- a/docs/translations/README.hi-IN.md +++ b/docs/translations/README.hi-IN.md @@ -28,7 +28,7 @@ graphify-out/ ├── graph.html इंटरेक्टिव ग्राफ — किसी भी ब्राउज़र में खोलें, नोड्स क्लिक करें, खोजें ├── GRAPH_REPORT.md गॉड नोड्स, आश्चर्यजनक कनेक्शन, सुझाए गए प्रश्न -├── graph.json स्थायी ग्राफ — हफ्तों बाद भी क्वेरी करें +├── graph.helix स्थायी ग्राफ — हफ्तों बाद भी क्वेरी करें └── cache/ SHA256 कैश — पुनः चलाने पर केवल बदली हुई फ़ाइलें प्रोसेस होती हैं ``` @@ -46,7 +46,7 @@ dist/ ## यह कैसे काम करता है -graphify तीन चरणों में चलता है। पहले, एक निर्धारक AST पास कोड फ़ाइलों से संरचना निकालता है — बिना किसी LLM के। दूसरे, वीडियो और ऑडियो फ़ाइलों को faster-whisper से स्थानीय रूप से ट्रांसक्राइब किया जाता है। तीसरे, Claude सबएजेंट दस्तावेज़ों, papers, छवियों और ट्रांसक्रिप्ट पर समानांतर में चलते हैं। परिणामों को NetworkX ग्राफ में मर्ज किया जाता है, Leiden कम्युनिटी डिटेक्शन से क्लस्टर किया जाता है, और इंटरेक्टिव HTML, क्वेरी करने योग्य JSON और एक ऑडिट रिपोर्ट के रूप में निर्यात किया जाता है। +graphify तीन चरणों में चलता है। पहले, एक निर्धारक AST पास कोड फ़ाइलों से संरचना निकालता है — बिना किसी LLM के। दूसरे, वीडियो और ऑडियो फ़ाइलों को faster-whisper से स्थानीय रूप से ट्रांसक्राइब किया जाता है। तीसरे, Claude सबएजेंट दस्तावेज़ों, papers, छवियों और ट्रांसक्रिप्ट पर समानांतर में चलते हैं। परिणामों को Helix ग्राफ में मर्ज किया जाता है, Leiden कम्युनिटी डिटेक्शन से क्लस्टर किया जाता है, और इंटरेक्टिव HTML, क्वेरी करने योग्य JSON और एक ऑडिट रिपोर्ट के रूप में निर्यात किया जाता है। **क्लस्टरिंग ग्राफ-टोपोलॉजी आधारित है — कोई embeddings नहीं।** Claude द्वारा निकाले गए सिमेंटिक समानता किनारे पहले से ग्राफ में हैं, इसलिए वे कम्युनिटी डिटेक्शन को सीधे प्रभावित करते हैं। diff --git a/docs/translations/README.hu-HU.md b/docs/translations/README.hu-HU.md index 14c24c1cb..0b241b38d 100644 --- a/docs/translations/README.hu-HU.md +++ b/docs/translations/README.hu-HU.md @@ -27,13 +27,13 @@ Teljesen multimodális. Adjon hozzá kódot, PDF-eket, markdownt, képernyőkép graphify-out/ ├── graph.html interaktív gráf — nyissa meg bármely böngészőben ├── GRAPH_REPORT.md isten-csúcspontok, meglepő kapcsolatok, javasolt kérdések -├── graph.json állandó gráf — hetekkel később is lekérdezhető +├── graph.helix állandó gráf — hetekkel később is lekérdezhető └── cache/ SHA256-gyorsítótár — ismételt futtatások csak a módosított fájlokat dolgozzák fel ``` ## Hogyan működik -A graphify három menetben dolgozik. Először egy determinisztikus AST-menet kinyeri a struktúrát a kódfájlokból LLM nélkül. Ezután a video- és hangfájlokat a faster-whisper segítségével helyben írja át. Végül a Claude alügynökök párhuzamosan futnak dokumentumokon, cikkeken, képeken és átiratokban. Az eredményeket egy NetworkX-gráfba olvasztja össze, Leiden-nel klaszterezik, és interaktív HTML-ként, lekérdezhető JSON-ként és auditjelentésként exportálja. +A graphify három menetben dolgozik. Először egy determinisztikus AST-menet kinyeri a struktúrát a kódfájlokból LLM nélkül. Ezután a video- és hangfájlokat a faster-whisper segítségével helyben írja át. Végül a Claude alügynökök párhuzamosan futnak dokumentumokon, cikkeken, képeken és átiratokban. Az eredményeket egy Helix-gráfba olvasztja össze, Leiden-nel klaszterezik, és interaktív HTML-ként, lekérdezhető JSON-ként és auditjelentésként exportálja. Minden kapcsolat `EXTRACTED`, `INFERRED` (megbízhatósági pontszámmal) vagy `AMBIGUOUS` feliratot kap. diff --git a/docs/translations/README.id-ID.md b/docs/translations/README.id-ID.md index 86691d5fa..25660eccc 100644 --- a/docs/translations/README.id-ID.md +++ b/docs/translations/README.id-ID.md @@ -27,13 +27,13 @@ Sepenuhnya multimodal. Tambahkan kode, PDF, markdown, tangkapan layar, diagram, graphify-out/ ├── graph.html graf interaktif — buka di browser mana saja ├── GRAPH_REPORT.md node dewa, koneksi mengejutkan, pertanyaan yang disarankan -├── graph.json graf persisten — dapat dikueri berminggu-minggu kemudian +├── graph.helix graf persisten — dapat dikueri berminggu-minggu kemudian └── cache/ cache SHA256 — pengulangan hanya memproses file yang berubah ``` ## Cara Kerja -graphify bekerja dalam tiga tahap. Pertama, tahap AST deterministik mengekstrak struktur dari file kode tanpa LLM. Kemudian file video dan audio ditranskrip secara lokal dengan faster-whisper. Terakhir, sub-agen Claude berjalan secara paralel pada dokumen, makalah, gambar, dan transkripsi. Hasilnya digabungkan ke dalam graf NetworkX, dikelompokkan dengan Leiden, dan diekspor sebagai HTML interaktif, JSON yang dapat dikueri, dan laporan audit. +graphify bekerja dalam tiga tahap. Pertama, tahap AST deterministik mengekstrak struktur dari file kode tanpa LLM. Kemudian file video dan audio ditranskrip secara lokal dengan faster-whisper. Terakhir, sub-agen Claude berjalan secara paralel pada dokumen, makalah, gambar, dan transkripsi. Hasilnya digabungkan ke dalam graf Helix, dikelompokkan dengan Leiden, dan diekspor sebagai HTML interaktif, JSON yang dapat dikueri, dan laporan audit. Setiap hubungan diberi label `EXTRACTED`, `INFERRED` (dengan skor kepercayaan), atau `AMBIGUOUS`. diff --git a/docs/translations/README.it-IT.md b/docs/translations/README.it-IT.md index 9cb1ccb91..1f789ffab 100644 --- a/docs/translations/README.it-IT.md +++ b/docs/translations/README.it-IT.md @@ -27,13 +27,13 @@ Completamente multimodale. Aggiungi codice, PDF, markdown, screenshot, diagrammi graphify-out/ ├── graph.html grafo interattivo — apri in qualsiasi browser ├── GRAPH_REPORT.md nodi dio, connessioni sorprendenti, domande suggerite -├── graph.json grafo persistente — interrogabile settimane dopo +├── graph.helix grafo persistente — interrogabile settimane dopo └── cache/ cache SHA256 — le riesecuzioni elaborano solo i file modificati ``` ## Come funziona -graphify esegue in tre passaggi. Prima, un passaggio AST deterministico estrae la struttura dai file di codice senza LLM. Poi, i file video e audio vengono trascritti localmente con faster-whisper. Infine, i subagenti Claude eseguono in parallelo su documenti, paper, immagini e trascrizioni. I risultati vengono uniti in un grafo NetworkX, raggruppati con Leiden e esportati come HTML interattivo, JSON interrogabile e report di audit. +graphify esegue in tre passaggi. Prima, un passaggio AST deterministico estrae la struttura dai file di codice senza LLM. Poi, i file video e audio vengono trascritti localmente con faster-whisper. Infine, i subagenti Claude eseguono in parallelo su documenti, paper, immagini e trascrizioni. I risultati vengono uniti in un grafo Helix, raggruppati con Leiden e esportati come HTML interattivo, JSON interrogabile e report di audit. Ogni relazione è etichettata `EXTRACTED`, `INFERRED` (con punteggio di confidenza) o `AMBIGUOUS`. diff --git a/docs/translations/README.ja-JP.md b/docs/translations/README.ja-JP.md index 467ff6839..332aadb9c 100644 --- a/docs/translations/README.ja-JP.md +++ b/docs/translations/README.ja-JP.md @@ -20,7 +20,7 @@ graphify-out/ ├── graph.html インタラクティブなグラフ - ノードをクリック、検索、コミュニティでフィルタ ├── GRAPH_REPORT.md ゴッドノード、意外なつながり、推奨される質問 -├── graph.json 永続化されたグラフ - 数週間後でも再読み込みなしでクエリ可能 +├── graph.helix 永続化されたグラフ - 数週間後でも再読み込みなしでクエリ可能 └── cache/ SHA256 キャッシュ - 再実行時は変更されたファイルのみ処理 ``` @@ -38,7 +38,7 @@ dist/ ## 仕組み -graphify は 2 パスで動作します。まず、決定論的な AST パスがコードファイルから構造(クラス、関数、インポート、コールグラフ、docstring、根拠コメント)を LLM なしで抽出します。次に、Claude サブエージェントがドキュメント、論文、画像に対して並列に実行され、概念、関係性、設計の根拠を抽出します。結果は NetworkX グラフにマージされ、Leiden コミュニティ検出でクラスタリングされ、インタラクティブ HTML、クエリ可能な JSON、平易な言葉の監査レポートとしてエクスポートされます。 +graphify は 2 パスで動作します。まず、決定論的な AST パスがコードファイルから構造(クラス、関数、インポート、コールグラフ、docstring、根拠コメント)を LLM なしで抽出します。次に、Claude サブエージェントがドキュメント、論文、画像に対して並列に実行され、概念、関係性、設計の根拠を抽出します。結果は Helix グラフにマージされ、Leiden コミュニティ検出でクラスタリングされ、インタラクティブ HTML、クエリ可能な JSON、平易な言葉の監査レポートとしてエクスポートされます。 **クラスタリングはグラフトポロジベース――埋め込みは使いません。** Leiden はエッジ密度によってコミュニティを見つけます。Claude が抽出する意味的類似性エッジ(`semantically_similar_to`、INFERRED とマーク)は既にグラフに含まれているため、コミュニティ検出に直接影響します。グラフ構造そのものが類似性シグナルであり――別途の埋め込みステップやベクターデータベースは不要です。 @@ -97,7 +97,7 @@ Codex ユーザーは並列抽出のために `~/.codex/config.toml` の `[featu 常時有効のフックは `GRAPH_REPORT.md` を表面化します――これはゴッドノード、コミュニティ、意外なつながりを 1 ページにまとめた要約です。アシスタントはファイル検索の前にこれを読み、キーワードマッチではなく構造に基づいてナビゲートします。これで日常的な質問のほとんどをカバーできます。 -`/graphify query`、`/graphify path`、`/graphify explain` はさらに深く踏み込みます:生の `graph.json` をホップごとに辿り、ノード間の正確なパスをトレースし、エッジレベルの詳細(関係タイプ、信頼度スコア、ソース位置)を表面化します。一般的なオリエンテーションではなく、特定の質問をグラフから答えさせたいときに使います。 +`/graphify query`、`/graphify path`、`/graphify explain` はさらに深く踏み込みます:生の `graph.helix` をホップごとに辿り、ノード間の正確なパスをトレースし、エッジレベルの詳細(関係タイプ、信頼度スコア、ソース位置)を表面化します。一般的なオリエンテーションではなく、特定の質問をグラフから答えさせたいときに使います。 こう考えてください:常時有効のフックはアシスタントに地図を与え、`/graphify` コマンドはその地図を正確にナビゲートさせます。 @@ -167,7 +167,7 @@ graphify droid install # AGENTS.md(Factory Droid) graphify query "アテンションとオプティマイザを結ぶものは?" graphify query "認証フローを表示" --dfs graphify query "CfgNode とは?" --budget 500 -graphify query "..." --graph path/to/graph.json +graphify query "..." --graph path/to/graph.helix ``` あらゆるファイルタイプの組み合わせで動作します: @@ -212,7 +212,7 @@ graphify query "..." --graph path/to/graph.json | graphify ソース + Transformer 論文 | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | | httpx(合成 Python ライブラリ) | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | -トークン削減はコーパスサイズに応じてスケールします。6 ファイルはいずれにせよコンテキストウィンドウに収まるため、そこでのグラフの価値は圧縮ではなく構造的明瞭さです。52 ファイル(コード + 論文 + 画像)では 71 倍以上が得られます。各 `worked/` フォルダには生の入力ファイルと実際の出力(`GRAPH_REPORT.md`、`graph.json`)があり、自分で実行して数字を検証できます。 +トークン削減はコーパスサイズに応じてスケールします。6 ファイルはいずれにせよコンテキストウィンドウに収まるため、そこでのグラフの価値は圧縮ではなく構造的明瞭さです。52 ファイル(コード + 論文 + 画像)では 71 倍以上が得られます。各 `worked/` フォルダには生の入力ファイルと実際の出力(`GRAPH_REPORT.md`、`graph.helix`)があり、自分で実行して数字を検証できます。 ## プライバシー @@ -220,7 +220,7 @@ graphify はドキュメント、論文、画像の意味的抽出のために ## 技術スタック -NetworkX + Leiden(graspologic) + tree-sitter + vis.js。意味的抽出は Claude(Claude Code)、GPT-4(Codex)、またはプラットフォームが実行するモデルを介して行われます。Neo4j は不要、サーバーも不要、完全にローカルで実行されます。 +Helix + Leiden(native Leiden) + tree-sitter + vis.js。意味的抽出は Claude(Claude Code)、GPT-4(Codex)、またはプラットフォームが実行するモデルを介して行われます。Neo4j は不要、サーバーも不要、完全にローカルで実行されます。 ## スター履歴 diff --git a/docs/translations/README.ko-KR.md b/docs/translations/README.ko-KR.md index 0fa46a211..b9b7d2e4f 100644 --- a/docs/translations/README.ko-KR.md +++ b/docs/translations/README.ko-KR.md @@ -20,7 +20,7 @@ graphify-out/ ├── graph.html 인터랙티브 그래프 - 노드 클릭, 검색, 커뮤니티별 필터 ├── GRAPH_REPORT.md 갓 노드, 의외의 연결, 추천 질문 -├── graph.json 영속 그래프 - 몇 주 후에도 재읽기 없이 쿼리 가능 +├── graph.helix 영속 그래프 - 몇 주 후에도 재읽기 없이 쿼리 가능 └── cache/ SHA256 캐시 - 재실행 시 변경된 파일만 처리 ``` @@ -38,7 +38,7 @@ dist/ ## 동작 원리 -graphify는 두 번의 패스로 실행됩니다. 첫 번째는 결정론적 AST 패스로, 코드 파일에서 구조(클래스, 함수, 임포트, 콜 그래프, docstring, 근거 주석)를 LLM 없이 추출합니다. 두 번째는 Claude 서브에이전트가 문서, 논문, 이미지에 대해 병렬로 실행되어 개념, 관계, 설계 근거를 추출합니다. 결과는 NetworkX 그래프로 병합되고, Leiden 커뮤니티 탐지로 클러스터링되며, 인터랙티브 HTML, 쿼리 가능한 JSON, 그리고 일반 언어 감사 보고서로 내보내집니다. +graphify는 두 번의 패스로 실행됩니다. 첫 번째는 결정론적 AST 패스로, 코드 파일에서 구조(클래스, 함수, 임포트, 콜 그래프, docstring, 근거 주석)를 LLM 없이 추출합니다. 두 번째는 Claude 서브에이전트가 문서, 논문, 이미지에 대해 병렬로 실행되어 개념, 관계, 설계 근거를 추출합니다. 결과는 Helix 그래프로 병합되고, Leiden 커뮤니티 탐지로 클러스터링되며, 인터랙티브 HTML, 쿼리 가능한 JSON, 그리고 일반 언어 감사 보고서로 내보내집니다. **클러스터링은 그래프 토폴로지 기반 — 임베딩을 사용하지 않습니다.** Leiden은 엣지 밀도를 기반으로 커뮤니티를 찾습니다. Claude가 추출하는 의미적 유사성 엣지(`semantically_similar_to`, INFERRED로 표시)는 이미 그래프에 포함되어 있으므로 커뮤니티 탐지에 직접 영향을 줍니다. 그래프 구조 자체가 유사성 신호이며 — 별도의 임베딩 단계나 벡터 데이터베이스가 필요하지 않습니다. @@ -103,13 +103,13 @@ Codex 사용자는 병렬 추출을 위해 `~/.codex/config.toml`의 `[features] 상시 작동 훅은 `GRAPH_REPORT.md`를 노출합니다 — 갓 노드, 커뮤니티, 의외의 연결을 한 페이지로 요약한 것입니다. 어시스턴트는 파일 검색 전에 이것을 읽으므로 키워드 매칭이 아닌 구조 기반으로 탐색합니다. 이것만으로 대부분의 일상적인 질문을 처리할 수 있습니다. -`/graphify query`, `/graphify path`, `/graphify explain`은 더 깊이 들어갑니다: 원시 `graph.json`을 홉 단위로 순회하고, 노드 간의 정확한 경로를 추적하며, 엣지 수준의 세부 정보(관계 유형, 신뢰도 점수, 소스 위치)를 보여줍니다. 일반적인 오리엔테이션이 아닌 그래프에서 특정 질문에 답하고 싶을 때 사용하세요. +`/graphify query`, `/graphify path`, `/graphify explain`은 더 깊이 들어갑니다: 원시 `graph.helix`을 홉 단위로 순회하고, 노드 간의 정확한 경로를 추적하며, 엣지 수준의 세부 정보(관계 유형, 신뢰도 점수, 소스 위치)를 보여줍니다. 일반적인 오리엔테이션이 아닌 그래프에서 특정 질문에 답하고 싶을 때 사용하세요. 이렇게 생각하면 됩니다: 상시 작동 훅은 어시스턴트에게 지도를 주고, `/graphify` 명령은 그 지도를 정확하게 탐색하게 합니다. -## `graph.json`을 LLM과 함께 사용하기 +## `graph.helix`을 LLM과 함께 사용하기 -`graph.json`은 프롬프트에 한 번에 전부 붙여넣기 위한 것이 아닙니다. 유용한 워크플로우는 다음과 같습니다: +`graph.helix`은 프롬프트에 한 번에 전부 붙여넣기 위한 것이 아닙니다. 유용한 워크플로우는 다음과 같습니다: 1. `graphify-out/GRAPH_REPORT.md`로 높은 수준의 개요를 파악합니다. 2. `graphify query`를 사용하여 답하려는 특정 질문에 대한 더 작은 서브그래프를 가져옵니다. @@ -118,8 +118,8 @@ Codex 사용자는 병렬 추출을 위해 `~/.codex/config.toml`의 `[features] 예를 들어, 프로젝트에서 graphify를 실행한 후: ```bash -graphify query "show the auth flow" --graph graphify-out/graph.json -graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json +graphify query "show the auth flow" --graph graphify-out/graph.helix +graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.helix ``` 출력에는 노드 레이블, 엣지 유형, 신뢰도 태그, 소스 파일, 소스 위치가 포함됩니다. 이는 LLM을 위한 좋은 중간 컨텍스트 블록이 됩니다: @@ -129,10 +129,10 @@ graphify query "what connects DigestAuth to Response?" --graph graphify-out/grap 가능한 경우 소스 파일을 인용하세요. ``` -어시스턴트가 도구 호출이나 MCP를 지원하는 경우, 텍스트를 붙여넣는 대신 그래프를 직접 사용하세요. graphify는 `graph.json`을 MCP 서버로 노출할 수 있습니다: +어시스턴트가 도구 호출이나 MCP를 지원하는 경우, 텍스트를 붙여넣는 대신 그래프를 직접 사용하세요. graphify는 `graph.helix`을 MCP 서버로 노출할 수 있습니다: ```bash -python -m graphify.serve graphify-out/graph.json +python -m graphify.serve graphify-out/graph.helix ``` 이를 통해 어시스턴트가 `query_graph`, `get_node`, `get_neighbors`, `shortest_path` 같은 반복 쿼리에 구조화된 그래프 접근을 할 수 있습니다. @@ -207,7 +207,7 @@ graphify trae-cn uninstall graphify query "어텐션과 옵티마이저를 연결하는 것은?" graphify query "인증 흐름 보기" --dfs graphify query "CfgNode이 뭐지?" --budget 500 -graphify query "..." --graph path/to/graph.json +graphify query "..." --graph path/to/graph.helix ``` 다양한 파일 유형의 조합과 함께 동작합니다: @@ -252,7 +252,7 @@ graphify query "..." --graph path/to/graph.json | graphify 소스 + Transformer 논문 | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | | httpx (합성 Python 라이브러리) | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | -토큰 축소는 코퍼스 크기에 비례하여 확장됩니다. 6개 파일은 어차피 컨텍스트 윈도우에 들어가므로, 그래프의 가치는 압축이 아닌 구조적 명확성에 있습니다. 52개 파일(코드 + 논문 + 이미지)에서는 71배 이상을 달성합니다. 각 `worked/` 폴더에는 원본 입력 파일과 실제 출력(`GRAPH_REPORT.md`, `graph.json`)이 있어 직접 실행하여 수치를 검증할 수 있습니다. +토큰 축소는 코퍼스 크기에 비례하여 확장됩니다. 6개 파일은 어차피 컨텍스트 윈도우에 들어가므로, 그래프의 가치는 압축이 아닌 구조적 명확성에 있습니다. 52개 파일(코드 + 논문 + 이미지)에서는 71배 이상을 달성합니다. 각 `worked/` 폴더에는 원본 입력 파일과 실제 출력(`GRAPH_REPORT.md`, `graph.helix`)이 있어 직접 실행하여 수치를 검증할 수 있습니다. ## 개인정보 보호 @@ -260,7 +260,7 @@ graphify는 문서, 논문, 이미지의 의미적 추출을 위해 파일 내 ## 기술 스택 -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. 의미적 추출은 Claude(Claude Code), GPT-4(Codex), 또는 플랫폼이 실행하는 모델을 통해 수행됩니다. Neo4j 불필요, 서버 불필요, 완전히 로컬에서 실행됩니다. +Helix + Leiden (native Leiden) + tree-sitter + vis.js. 의미적 추출은 Claude(Claude Code), GPT-4(Codex), 또는 플랫폼이 실행하는 모델을 통해 수행됩니다. Neo4j 불필요, 서버 불필요, 완전히 로컬에서 실행됩니다. ## 다음 계획 diff --git a/docs/translations/README.nl-NL.md b/docs/translations/README.nl-NL.md index 2dd8f834f..2dc675067 100644 --- a/docs/translations/README.nl-NL.md +++ b/docs/translations/README.nl-NL.md @@ -27,13 +27,13 @@ Volledig multimodaal. Voeg code, PDF's, markdown, schermafbeeldingen, diagrammen graphify-out/ ├── graph.html interactieve graaf — open in elke browser ├── GRAPH_REPORT.md godknooppunten, verrassende verbindingen, voorgestelde vragen -├── graph.json persistente graaf — weken later opvraagbaar +├── graph.helix persistente graaf — weken later opvraagbaar └── cache/ SHA256-cache — herhaalde runs verwerken alleen gewijzigde bestanden ``` ## Hoe het werkt -graphify werkt in drie passes. Eerst extraheert een deterministische AST-pass structuur uit codebestanden zonder LLM. Vervolgens worden video- en audiobestanden lokaal getranscribeerd met faster-whisper. Ten slotte werken Claude-subagenten parallel over documenten, papers, afbeeldingen en transcripties. De resultaten worden samengevoegd in een NetworkX-graaf, geclusterd met Leiden en geëxporteerd als interactieve HTML, opvraagbare JSON en een auditrapport. +graphify werkt in drie passes. Eerst extraheert een deterministische AST-pass structuur uit codebestanden zonder LLM. Vervolgens worden video- en audiobestanden lokaal getranscribeerd met faster-whisper. Ten slotte werken Claude-subagenten parallel over documenten, papers, afbeeldingen en transcripties. De resultaten worden samengevoegd in een Helix-graaf, geclusterd met Leiden en geëxporteerd als interactieve HTML, opvraagbare JSON en een auditrapport. Elke relatie is gelabeld als `EXTRACTED`, `INFERRED` (met betrouwbaarheidsscore) of `AMBIGUOUS`. diff --git a/docs/translations/README.no-NO.md b/docs/translations/README.no-NO.md index 5bfcd59be..589a3ad2b 100644 --- a/docs/translations/README.no-NO.md +++ b/docs/translations/README.no-NO.md @@ -27,13 +27,13 @@ Fullt multimodal. Legg til kode, PDF-er, markdown, skjermbilder, diagrammer, whi graphify-out/ ├── graph.html interaktiv graf — åpne i en hvilken som helst nettleser ├── GRAPH_REPORT.md gudnoder, overraskende forbindelser, foreslåtte spørsmål -├── graph.json vedvarende graf — forespørselbar uker senere +├── graph.helix vedvarende graf — forespørselbar uker senere └── cache/ SHA256-cache — gjentatte kjøringer behandler bare endrede filer ``` ## Hvordan det fungerer -graphify arbeider i tre gjennomganger. Først ekstraherer et deterministisk AST-gjennomgang struktur fra kodefiler uten LLM. Deretter transkriberes video- og lydfiler lokalt med faster-whisper. Til slutt kjører Claude-underagenter parallelt på dokumenter, artikler, bilder og transkripsjoner. Resultatene slås sammen i en NetworkX-graf, klynges med Leiden og eksporteres som interaktiv HTML, forespørselbar JSON og revisjonsrapport. +graphify arbeider i tre gjennomganger. Først ekstraherer et deterministisk AST-gjennomgang struktur fra kodefiler uten LLM. Deretter transkriberes video- og lydfiler lokalt med faster-whisper. Til slutt kjører Claude-underagenter parallelt på dokumenter, artikler, bilder og transkripsjoner. Resultatene slås sammen i en Helix-graf, klynges med Leiden og eksporteres som interaktiv HTML, forespørselbar JSON og revisjonsrapport. Hver relasjon er merket `EXTRACTED`, `INFERRED` (med konfidenspoeng) eller `AMBIGUOUS`. diff --git a/docs/translations/README.pl-PL.md b/docs/translations/README.pl-PL.md index 555dd4c03..0bbc47c3a 100644 --- a/docs/translations/README.pl-PL.md +++ b/docs/translations/README.pl-PL.md @@ -27,13 +27,13 @@ W pełni multimodalny. Dodaj kod, PDF, markdown, zrzuty ekranu, diagramy, zdjęc graphify-out/ ├── graph.html interaktywny graf — otwórz w dowolnej przeglądarce ├── GRAPH_REPORT.md węzły boga, zaskakujące połączenia, sugerowane pytania -├── graph.json trwały graf — zapytaj tygodnie później +├── graph.helix trwały graf — zapytaj tygodnie później └── cache/ cache SHA256 — ponowne uruchomienia przetwarzają tylko zmienione pliki ``` ## Jak to działa -graphify działa w trzech przebiegach. Najpierw deterministyczny przebieg AST wyodrębnia strukturę z plików kodu bez LLM. Następnie pliki wideo i audio są transkrybowane lokalnie za pomocą faster-whisper. Na koniec subagenci Claude działają równolegle na dokumentach, artykułach, obrazach i transkrypcjach. Wyniki są łączone w graf NetworkX, grupowane za pomocą Leiden i eksportowane jako interaktywny HTML, JSON i raport audytu. +graphify działa w trzech przebiegach. Najpierw deterministyczny przebieg AST wyodrębnia strukturę z plików kodu bez LLM. Następnie pliki wideo i audio są transkrybowane lokalnie za pomocą faster-whisper. Na koniec subagenci Claude działają równolegle na dokumentach, artykułach, obrazach i transkrypcjach. Wyniki są łączone w graf Helix, grupowane za pomocą Leiden i eksportowane jako interaktywny HTML, JSON i raport audytu. Każda relacja jest oznaczona `EXTRACTED`, `INFERRED` (z wynikiem pewności) lub `AMBIGUOUS`. diff --git a/docs/translations/README.pt-BR.md b/docs/translations/README.pt-BR.md index 37e7d3f6e..2364b755b 100644 --- a/docs/translations/README.pt-BR.md +++ b/docs/translations/README.pt-BR.md @@ -28,7 +28,7 @@ Totalmente multimodal. Adicione código, PDFs, markdown, capturas de tela, diagr graphify-out/ ├── graph.html grafo interativo — abrir em qualquer navegador, clicar em nós, pesquisar ├── GRAPH_REPORT.md nós deus, conexões surpreendentes, perguntas sugeridas -├── graph.json grafo persistente — consultar semanas depois sem reler +├── graph.helix grafo persistente — consultar semanas depois sem reler └── cache/ cache SHA256 — re-execuções processam apenas arquivos modificados ``` @@ -46,7 +46,7 @@ Mesma sintaxe do `.gitignore`. ## Como funciona -graphify executa em três passes. Primeiro, uma passagem AST determinística extrai estrutura de arquivos de código (classes, funções, importações, grafos de chamadas, docstrings, comentários de justificativa) sem LLM. Segundo, arquivos de vídeo e áudio são transcritos localmente com faster-whisper. Terceiro, subagentes Claude executam em paralelo sobre documentos, papers, imagens e transcrições para extrair conceitos, relações e justificativas de design. Os resultados são mesclados em um grafo NetworkX, agrupados com detecção de comunidades Leiden, e exportados como HTML interativo, JSON consultável e um relatório de auditoria em linguagem natural. +graphify executa em três passes. Primeiro, uma passagem AST determinística extrai estrutura de arquivos de código (classes, funções, importações, grafos de chamadas, docstrings, comentários de justificativa) sem LLM. Segundo, arquivos de vídeo e áudio são transcritos localmente com faster-whisper. Terceiro, subagentes Claude executam em paralelo sobre documentos, papers, imagens e transcrições para extrair conceitos, relações e justificativas de design. Os resultados são mesclados em um grafo Helix, agrupados com detecção de comunidades Leiden, e exportados como HTML interativo, JSON consultável e um relatório de auditoria em linguagem natural. **O clustering é baseado em topologia de grafo — sem embeddings.** Leiden encontra comunidades por densidade de arestas. As arestas de similaridade semântica que Claude extrai (`semantically_similar_to`, marcadas INFERRED) já estão no grafo. A estrutura do grafo é o sinal de similaridade — nenhum passo de embedding separado ou banco de dados vetorial é necessário. @@ -157,7 +157,7 @@ graphify envia conteúdo de arquivos para a API do modelo do seu assistente IA p ## Stack técnico -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Extração semântica via Claude, GPT-4 ou o modelo da sua plataforma. Transcrição de vídeo via faster-whisper + yt-dlp (opcional). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Extração semântica via Claude, GPT-4 ou o modelo da sua plataforma. Transcrição de vídeo via faster-whisper + yt-dlp (opcional). ## Construído sobre graphify — Penpax diff --git a/docs/translations/README.ro-RO.md b/docs/translations/README.ro-RO.md index 159354118..8faa94f0d 100644 --- a/docs/translations/README.ro-RO.md +++ b/docs/translations/README.ro-RO.md @@ -27,13 +27,13 @@ Complet multimodal. Adăugați cod, PDF-uri, markdown, capturi de ecran, diagram graphify-out/ ├── graph.html graf interactiv — deschideți în orice browser ├── GRAPH_REPORT.md noduri-zeu, conexiuni surprinzătoare, întrebări sugerate -├── graph.json graf persistent — interogabil săptămâni mai târziu +├── graph.helix graf persistent — interogabil săptămâni mai târziu └── cache/ cache SHA256 — rulările repetate procesează doar fișierele modificate ``` ## Cum funcționează -graphify lucrează în trei treceri. Mai întâi, o trecere AST deterministă extrage structura din fișierele de cod fără LLM. Apoi fișierele video și audio sunt transcrise local cu faster-whisper. În final, sub-agenții Claude rulează în paralel pe documente, lucrări, imagini și transcrieri. Rezultatele sunt îmbinate într-un graf NetworkX, grupate cu Leiden și exportate ca HTML interactiv, JSON interogabil și raport de audit. +graphify lucrează în trei treceri. Mai întâi, o trecere AST deterministă extrage structura din fișierele de cod fără LLM. Apoi fișierele video și audio sunt transcrise local cu faster-whisper. În final, sub-agenții Claude rulează în paralel pe documente, lucrări, imagini și transcrieri. Rezultatele sunt îmbinate într-un graf Helix, grupate cu Leiden și exportate ca HTML interactiv, JSON interogabil și raport de audit. Fiecare relație este etichetată `EXTRACTED`, `INFERRED` (cu scor de încredere) sau `AMBIGUOUS`. diff --git a/docs/translations/README.ru-RU.md b/docs/translations/README.ru-RU.md index 7518aea41..dd0d0abcf 100644 --- a/docs/translations/README.ru-RU.md +++ b/docs/translations/README.ru-RU.md @@ -28,7 +28,7 @@ graphify-out/ ├── graph.html интерактивный граф — открыть в браузере, кликать по узлам, искать, фильтровать ├── GRAPH_REPORT.md бог-узлы, неожиданные связи, предлагаемые вопросы -├── graph.json постоянный граф — запрашивать через недели без повторного чтения +├── graph.helix постоянный граф — запрашивать через недели без повторного чтения └── cache/ SHA256-кэш — повторные запуски обрабатывают только изменённые файлы ``` @@ -46,7 +46,7 @@ dist/ ## Как это работает -graphify работает в три прохода. Сначала детерминированный AST-проход извлекает структуру из файлов кода (классы, функции, импорты, графы вызовов, docstrings, комментарии с обоснованием) — без LLM. Затем видео и аудиофайлы транскрибируются локально с faster-whisper. Наконец, Claude-субагенты запускаются параллельно над документами, статьями, изображениями и транскриптами для извлечения концепций, связей и обоснований дизайна. Результаты объединяются в граф NetworkX, кластеризуются с помощью Leiden-детекции сообществ и экспортируются как интерактивный HTML, запрашиваемый JSON и аудит-отчёт на естественном языке. +graphify работает в три прохода. Сначала детерминированный AST-проход извлекает структуру из файлов кода (классы, функции, импорты, графы вызовов, docstrings, комментарии с обоснованием) — без LLM. Затем видео и аудиофайлы транскрибируются локально с faster-whisper. Наконец, Claude-субагенты запускаются параллельно над документами, статьями, изображениями и транскриптами для извлечения концепций, связей и обоснований дизайна. Результаты объединяются в граф Helix, кластеризуются с помощью Leiden-детекции сообществ и экспортируются как интерактивный HTML, запрашиваемый JSON и аудит-отчёт на естественном языке. **Кластеризация основана на топологии графа — без эмбеддингов.** Leiden находит сообщества по плотности рёбер. Рёбра семантического сходства, извлечённые Claude (`semantically_similar_to`, помечены как INFERRED), уже в графе. Структура графа — это сигнал сходства. Отдельный шаг с эмбеддингами или векторная база данных не нужны. @@ -157,7 +157,7 @@ graphify отправляет содержимое файлов в API моде ## Технологический стек -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Семантическое извлечение через Claude, GPT-4 или модель вашей платформы. Транскрипция видео через faster-whisper + yt-dlp (опционально). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Семантическое извлечение через Claude, GPT-4 или модель вашей платформы. Транскрипция видео через faster-whisper + yt-dlp (опционально). ## Построено на graphify — Penpax diff --git a/docs/translations/README.sv-SE.md b/docs/translations/README.sv-SE.md index 63e015d95..d184051cb 100644 --- a/docs/translations/README.sv-SE.md +++ b/docs/translations/README.sv-SE.md @@ -27,13 +27,13 @@ Helt multimodal. Lägg till kod, PDF:er, markdown, skärmdumpar, diagram, whiteb graphify-out/ ├── graph.html interaktivt diagram — öppna i valfri webbläsare ├── GRAPH_REPORT.md gudnoder, överraskande kopplingar, föreslagna frågor -├── graph.json beständigt diagram — kan frågas veckor senare +├── graph.helix beständigt diagram — kan frågas veckor senare └── cache/ SHA256-cache — upprepade körningar behandlar bara ändrade filer ``` ## Hur det fungerar -graphify arbetar i tre pass. Först extraherar ett deterministiskt AST-pass struktur från kodfiler utan LLM. Sedan transkriberas video- och ljudfiler lokalt med faster-whisper. Slutligen kör Claude-subagenter parallellt på dokument, papper, bilder och transkriptioner. Resultaten slås samman i ett NetworkX-diagram, klustras med Leiden och exporteras som interaktiv HTML, frågebar JSON och revisionsrapport. +graphify arbetar i tre pass. Först extraherar ett deterministiskt AST-pass struktur från kodfiler utan LLM. Sedan transkriberas video- och ljudfiler lokalt med faster-whisper. Slutligen kör Claude-subagenter parallellt på dokument, papper, bilder och transkriptioner. Resultaten slås samman i ett Helix-diagram, klustras med Leiden och exporteras som interaktiv HTML, frågebar JSON och revisionsrapport. Varje relation är märkt `EXTRACTED`, `INFERRED` (med konfidenspoäng) eller `AMBIGUOUS`. diff --git a/docs/translations/README.th-TH.md b/docs/translations/README.th-TH.md index 8c1b05969..86612e82b 100644 --- a/docs/translations/README.th-TH.md +++ b/docs/translations/README.th-TH.md @@ -27,13 +27,13 @@ graphify-out/ ├── graph.html กราฟแบบโต้ตอบ — เปิดในเบราว์เซอร์ใดก็ได้ ├── GRAPH_REPORT.md โหนดพระเจ้า, การเชื่อมต่อที่น่าประหลาดใจ, คำถามที่แนะนำ -├── graph.json กราฟถาวร — สามารถสืบค้นได้หลายสัปดาห์ต่อมา +├── graph.helix กราฟถาวร — สามารถสืบค้นได้หลายสัปดาห์ต่อมา └── cache/ SHA256-cache — การรันซ้ำประมวลผลเฉพาะไฟล์ที่เปลี่ยนแปลง ``` ## วิธีการทำงาน -graphify ทำงานใน 3 รอบ ก่อนอื่น AST pass แบบ deterministic ดึงโครงสร้างจากไฟล์โค้ดโดยไม่ต้องใช้ LLM จากนั้นไฟล์วิดีโอและเสียงถูกถอดเสียงในเครื่องด้วย faster-whisper สุดท้าย Claude sub-agent ทำงานแบบขนานกันบนเอกสาร, งานวิจัย, รูปภาพ และบทถอดเสียง ผลลัพธ์ถูกรวมเข้ากับกราฟ NetworkX, จัดกลุ่มด้วย Leiden และส่งออกเป็น HTML แบบโต้ตอบ, JSON ที่สืบค้นได้ และรายงานการตรวจสอบ +graphify ทำงานใน 3 รอบ ก่อนอื่น AST pass แบบ deterministic ดึงโครงสร้างจากไฟล์โค้ดโดยไม่ต้องใช้ LLM จากนั้นไฟล์วิดีโอและเสียงถูกถอดเสียงในเครื่องด้วย faster-whisper สุดท้าย Claude sub-agent ทำงานแบบขนานกันบนเอกสาร, งานวิจัย, รูปภาพ และบทถอดเสียง ผลลัพธ์ถูกรวมเข้ากับกราฟ Helix, จัดกลุ่มด้วย Leiden และส่งออกเป็น HTML แบบโต้ตอบ, JSON ที่สืบค้นได้ และรายงานการตรวจสอบ ความสัมพันธ์แต่ละอย่างถูกติดป้าย `EXTRACTED`, `INFERRED` (พร้อมคะแนนความเชื่อมั่น) หรือ `AMBIGUOUS` diff --git a/docs/translations/README.tr-TR.md b/docs/translations/README.tr-TR.md index 77f30742e..734f42772 100644 --- a/docs/translations/README.tr-TR.md +++ b/docs/translations/README.tr-TR.md @@ -27,13 +27,13 @@ Tamamen çok modlu. Kod, PDF, markdown, ekran görüntüleri, diyagramlar, beyaz graphify-out/ ├── graph.html etkileşimli grafik — herhangi bir tarayıcıda açın ├── GRAPH_REPORT.md tanrı düğümleri, şaşırtıcı bağlantılar, önerilen sorular -├── graph.json kalıcı grafik — haftalar sonra sorgulanabilir +├── graph.helix kalıcı grafik — haftalar sonra sorgulanabilir └── cache/ SHA256 önbelleği — tekrarlanan çalışmalar yalnızca değiştirilen dosyaları işler ``` ## Nasıl çalışır -graphify üç geçişte çalışır. Önce deterministik bir AST geçişi, LLM olmadan kod dosyalarından yapı çıkarır. Ardından video ve ses dosyaları faster-whisper ile yerel olarak transkribe edilir. Son olarak Claude alt ajanları belgeler, makaleler, görüntüler ve transkriptler üzerinde paralel olarak çalışır. Sonuçlar bir NetworkX grafiğinde birleştirilir, Leiden ile kümelenir ve etkileşimli HTML, sorgulanabilir JSON ve denetim raporu olarak dışa aktarılır. +graphify üç geçişte çalışır. Önce deterministik bir AST geçişi, LLM olmadan kod dosyalarından yapı çıkarır. Ardından video ve ses dosyaları faster-whisper ile yerel olarak transkribe edilir. Son olarak Claude alt ajanları belgeler, makaleler, görüntüler ve transkriptler üzerinde paralel olarak çalışır. Sonuçlar bir Helix grafiğinde birleştirilir, Leiden ile kümelenir ve etkileşimli HTML, sorgulanabilir JSON ve denetim raporu olarak dışa aktarılır. Her ilişki `EXTRACTED`, `INFERRED` (güven puanıyla) veya `AMBIGUOUS` olarak etiketlenir. diff --git a/docs/translations/README.uk-UA.md b/docs/translations/README.uk-UA.md index 2fa632544..528e4599f 100644 --- a/docs/translations/README.uk-UA.md +++ b/docs/translations/README.uk-UA.md @@ -37,7 +37,7 @@ graphify-out/ ├── graph.html відкрийте в будь-якому браузері — клікайте по вузлах, фільтруйте, шукайте ├── GRAPH_REPORT.md основне: ключові концепції, неочікувані зв’язки, запропоновані запитання -└── graph.json повний граф — запитуйте його будь-коли без повторного перечитування ваших файлів +└── graph.helix повний граф — запитуйте його будь-коли без повторного перечитування ваших файлів ``` Для читабельної сторінки архітектури з діаграмами викликів Mermaid виконайте: @@ -291,7 +291,7 @@ graphify-out/cost.json # лише локальний **Робочий процес:** 1. Одна людина запускає `/graphify .` і комітить `graphify-out/`. 2. Усі виконують pull — їхній асистент одразу читає граф. -3. Запустіть `graphify hook install` для автоматичного перебудування після кожного коміту (лише AST, без витрат API). Це також налаштовує git merge driver, щоб `graph.json` ніколи не залишався з маркерами конфліктів — два розробники, що комітять одночасно, отримають автоматично об'єднані графи. +3. Запустіть `graphify hook install` для автоматичного перебудування після кожного коміту (лише AST, без витрат API). Це також налаштовує git merge driver, щоб `graph.helix` ніколи не залишався з маркерами конфліктів — два розробники, що комітять одночасно, отримають автоматично об'єднані графи. 4. Коли документи або статті змінюються, запустіть `/graphify --update`, щоб оновити ці вузли. --- @@ -301,13 +301,13 @@ graphify-out/cost.json # лише локальний ```bash # запит до графу з терміналу graphify query "покажи потік автентифікації" -graphify query "що пов'язує DigestAuth з Response?" --graph graphify-out/graph.json +graphify query "що пов'язує DigestAuth з Response?" --graph graphify-out/graph.helix # відкрити граф як MCP-сервер (для повторного доступу через інструменти) -python -m graphify.serve graphify-out/graph.json +python -m graphify.serve graphify-out/graph.helix # зареєструвати в Kimi Code: -kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.json +kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.helix ``` MCP-сервер надає асистенту структурований доступ: `query_graph`, `get_node`, `get_neighbors`, `shortest_path`, `list_prs`, `get_pr_impact`, `triage_prs`. @@ -394,8 +394,8 @@ graphify cluster-only ./my-project --no-viz graphify query "..." ``` -**`graph.json` має маркери конфліктів після одночасного коміту двох розробників** -Запустіть `graphify hook install` — це налаштовує git merge driver, який автоматично об'єднує `graph.json`, щоб конфліктів ніколи не виникало. +**`graph.helix` має маркери конфліктів після одночасного коміту двох розробників** +Запустіть `graphify hook install` — це налаштовує git merge driver, який автоматично об'єднує `graph.helix`, щоб конфліктів ніколи не виникало. **Вилучення повертає порожні вузли/ребра для документів або PDF** Документи та PDF потребують LLM-виклику. Перевірте, що API-ключ встановлено і backend правильний: @@ -479,7 +479,7 @@ graphify extract ./docs --max-concurrency 2 # менше паралельни graphify extract ./docs --api-timeout 900 # довший HTTP тайм-аут для повільних локальних моделей (типово 600с) graphify extract ./docs --google-workspace # експортувати .gdoc/.gsheet/.gslides через gws перед витягуванням graphify extract ./docs --no-cluster # лише сире витягування, пропустити кластеризацію -graphify extract ./docs --force # перезаписати graph.json навіть якщо новий граф має менше вузлів (використовуйте після рефакторингу або для очищення фантомних дублікатів) +graphify extract ./docs --force # перезаписати graph.helix навіть якщо новий граф має менше вузлів (використовуйте після рефакторингу або для очищення фантомних дублікатів) graphify extract ./docs --dedup-llm # LLM-арбітр для неоднозначних пар сутностей (використовує той самий API-ключ) graphify extract ./docs --global --as myrepo # витягнути і зареєструвати в крос-проектний глобальний граф GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # підвищити ліміт виводу для щільних корпусів @@ -489,7 +489,7 @@ graphify export callflow-html --max-sections 8 # обмежити кіль graphify export callflow-html --output docs/arch.html graphify export callflow-html ./some-repo/graphify-out -graphify global add graphify-out/graph.json myrepo # зареєструвати граф проекту в ~/.graphify/global.json +graphify global add graphify-out/graph.helix myrepo # зареєструвати граф проекту в ~/.graphify/global.json graphify global remove myrepo # видалити проект з глобального графу graphify global list # показати всі зареєстровані репо + кількість вузлів/ребер graphify global path # вивести шлях до файлу глобального графу @@ -512,7 +512,7 @@ graphify update ./src graphify update ./src --no-cluster # пропустити рекластеризацію, записати лише сирий AST граф graphify update ./src --force # перезаписати навіть якщо новий граф має менше вузлів graphify cluster-only ./my-project -graphify cluster-only ./my-project --graph path/to/graph.json # власне розташування графу +graphify cluster-only ./my-project --graph path/to/graph.helix # власне розташування графу graphify cluster-only ./my-project --resolution 1.5 # більше, менших спільнот graphify cluster-only ./my-project --exclude-hubs 99 # виключити вузли p99 ступеня з розбиття ``` diff --git a/docs/translations/README.uz-UZ.md b/docs/translations/README.uz-UZ.md index 0e5f6e745..20c68ab0f 100644 --- a/docs/translations/README.uz-UZ.md +++ b/docs/translations/README.uz-UZ.md @@ -28,7 +28,7 @@ To'liq multimodal. Kod, PDF, markdown, ekran tasvirlari, diagrammalar, doska sur graphify-out/ ├── graph.html interaktiv graf — brauzerda oching, tugunlarni bosing, qidiring, filtrlang ├── GRAPH_REPORT.md god-tugunlar, kutilmagan aloqalar, taklif qilingan savollar -├── graph.json doimiy graf — haftalardan keyin qayta o'qimasdan so'rov qiling +├── graph.helix doimiy graf — haftalardan keyin qayta o'qimasdan so'rov qiling └── cache/ SHA256-kesh — qayta ishga tushirish faqat o'zgargan fayllarni qayta ishlaydi ``` @@ -46,7 +46,7 @@ Sintaksisi `.gitignore` ga o'xshash. ## Qanday ishlaydi -graphify uch bosqichda ishlaydi. Birinchi navbatda, deterministik AST bosqichi kod fayllaridan tuzilmani chiqaradi (klasslar, funksiyalar, importlar, chaqiruv graflari, docstring lar, sabab izohlari) — LLM ishtirokisiz. Keyin video va audio fayllar faster-whisper yordamida mahalliy ravishda transkripsiya qilinadi. Nihoyat, Claude subagentlari hujjatlar, maqolalar, tasvirlar va transkriptlar ustida parallel ishlab, tushunchalar, aloqalar va dizayn asoslarini chiqarib oladi. Natijalar NetworkX grafiga birlashtiriladi, Leiden hamjamiyat aniqlash algoritmi bilan klasterlanadi va interaktiv HTML, so'rov qilinadigan JSON hamda tabiiy tildagi audit hisoboti sifatida eksport qilinadi. +graphify uch bosqichda ishlaydi. Birinchi navbatda, deterministik AST bosqichi kod fayllaridan tuzilmani chiqaradi (klasslar, funksiyalar, importlar, chaqiruv graflari, docstring lar, sabab izohlari) — LLM ishtirokisiz. Keyin video va audio fayllar faster-whisper yordamida mahalliy ravishda transkripsiya qilinadi. Nihoyat, Claude subagentlari hujjatlar, maqolalar, tasvirlar va transkriptlar ustida parallel ishlab, tushunchalar, aloqalar va dizayn asoslarini chiqarib oladi. Natijalar Helix grafiga birlashtiriladi, Leiden hamjamiyat aniqlash algoritmi bilan klasterlanadi va interaktiv HTML, so'rov qilinadigan JSON hamda tabiiy tildagi audit hisoboti sifatida eksport qilinadi. **Klasterlash graf topologiyasiga asoslangan — embedding ishlatilmaydi.** Leiden hamjamiyatlarni qirralar zichligi bo'yicha topadi. Claude tomonidan chiqarilgan semantik o'xshashlik qirralari (`semantically_similar_to`, INFERRED deb belgilangan) allaqachon grafda. Graf tuzilmasining o'zi o'xshashlik signali. Alohida embedding bosqichi yoki vektor ma'lumotlar bazasi shart emas. @@ -157,7 +157,7 @@ graphify hujjatlar, maqolalar va tasvirlardan semantik chiqarish uchun fayl mazm ## Texnologiyalar to'plami -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantik chiqarish Claude, GPT-4 yoki sizning platformangiz modeli orqali. Video transkripsiyasi faster-whisper + yt-dlp orqali (opsional). +Helix + Leiden (native Leiden) + tree-sitter + vis.js. Semantik chiqarish Claude, GPT-4 yoki sizning platformangiz modeli orqali. Video transkripsiyasi faster-whisper + yt-dlp orqali (opsional). ## graphify ustida qurilgan — Penpax diff --git a/docs/translations/README.vi-VN.md b/docs/translations/README.vi-VN.md index 5c44dc6aa..f7e39468d 100644 --- a/docs/translations/README.vi-VN.md +++ b/docs/translations/README.vi-VN.md @@ -27,13 +27,13 @@ Hoàn toàn đa phương thức. Thêm code, PDF, markdown, ảnh chụp màn h graphify-out/ ├── graph.html đồ thị tương tác — mở trong bất kỳ trình duyệt nào ├── GRAPH_REPORT.md nút thần, kết nối bất ngờ, câu hỏi được đề xuất -├── graph.json đồ thị liên tục — có thể truy vấn sau nhiều tuần +├── graph.helix đồ thị liên tục — có thể truy vấn sau nhiều tuần └── cache/ bộ nhớ đệm SHA256 — các lần chạy lại chỉ xử lý các tệp đã thay đổi ``` ## Cách hoạt động -graphify hoạt động theo ba lần duyệt. Đầu tiên, một lần duyệt AST xác định trích xuất cấu trúc từ các tệp code mà không cần LLM. Sau đó, các tệp video và âm thanh được phiên âm cục bộ bằng faster-whisper. Cuối cùng, các sub-agent Claude chạy song song trên các tài liệu, bài báo, hình ảnh và bản phiên âm. Kết quả được hợp nhất vào đồ thị NetworkX, phân cụm với Leiden và xuất dưới dạng HTML tương tác, JSON có thể truy vấn và báo cáo kiểm tra. +graphify hoạt động theo ba lần duyệt. Đầu tiên, một lần duyệt AST xác định trích xuất cấu trúc từ các tệp code mà không cần LLM. Sau đó, các tệp video và âm thanh được phiên âm cục bộ bằng faster-whisper. Cuối cùng, các sub-agent Claude chạy song song trên các tài liệu, bài báo, hình ảnh và bản phiên âm. Kết quả được hợp nhất vào đồ thị Helix, phân cụm với Leiden và xuất dưới dạng HTML tương tác, JSON có thể truy vấn và báo cáo kiểm tra. Mỗi mối quan hệ được gắn nhãn `EXTRACTED`, `INFERRED` (với điểm tin cậy) hoặc `AMBIGUOUS`. diff --git a/docs/translations/README.zh-CN.md b/docs/translations/README.zh-CN.md index e41832293..0be06a10c 100644 --- a/docs/translations/README.zh-CN.md +++ b/docs/translations/README.zh-CN.md @@ -19,13 +19,13 @@ graphify-out/ ├── graph.html 可交互图谱:可点节点、搜索、按社区过滤 ├── GRAPH_REPORT.md God nodes、意外连接、建议提问 -├── graph.json 持久化图谱:数周后仍可查询,无需重新读原始文件 +├── graph.helix 持久化图谱:数周后仍可查询,无需重新读原始文件 └── cache/ SHA256 缓存:重复运行时只处理变更过的文件 ``` ## 工作原理 -graphify 分两轮执行。第一轮是确定性的 AST 提取,对代码文件做结构分析(类、函数、导入、调用图、docstring、解释性注释),这一轮不需要 LLM。第二轮会并行调用 Claude 子代理处理文档、论文和图片,从中提取概念、关系和设计动机。最后把两边结果合并到一个 NetworkX 图里,用 Leiden 社区发现算法做聚类,并导出成可交互 HTML、可查询 JSON,以及一份人类可读的审计报告。 +graphify 分两轮执行。第一轮是确定性的 AST 提取,对代码文件做结构分析(类、函数、导入、调用图、docstring、解释性注释),这一轮不需要 LLM。第二轮会并行调用 Claude 子代理处理文档、论文和图片,从中提取概念、关系和设计动机。最后把两边结果合并到一个 Helix 图里,用 Leiden 社区发现算法做聚类,并导出成可交互 HTML、可查询 JSON,以及一份人类可读的审计报告。 **聚类是基于图拓扑完成的,不依赖 embeddings。** Leiden 按边密度发现社区。Claude 抽取出的语义相似边(`semantically_similar_to`,标记为 `INFERRED`)本来就存在于图中,所以会直接影响社区划分。图结构本身就是相似性信号,不需要额外的 embedding 步骤,也不需要向量数据库。 @@ -93,7 +93,7 @@ Codex 用户还需要在 `~/.codex/config.toml` 的 `[features]` 下打开 `mult 常驻 hook 会优先暴露 `GRAPH_REPORT.md` —— 这是一页式总结,包含 god nodes、社区结构和意外连接。你的助手在搜索文件前会先读它,因此会按结构导航,而不是按关键字乱搜。这已经能覆盖大部分日常问题。 -`/graphify query`、`/graphify path` 和 `/graphify explain` 会更深入:它们会逐跳遍历底层 `graph.json`,追踪节点之间的精确路径,并展示边级别细节(关系类型、置信度、源位置)。当你想从图谱里精确回答某个问题,而不仅仅是获得整体感知时,就该用这些命令。 +`/graphify query`、`/graphify path` 和 `/graphify explain` 会更深入:它们会逐跳遍历底层 `graph.helix`,追踪节点之间的精确路径,并展示边级别细节(关系类型、置信度、源位置)。当你想从图谱里精确回答某个问题,而不仅仅是获得整体感知时,就该用这些命令。 可以这样理解:常驻 hook 是先给助手一张地图,`/graphify` 这几个命令则是让它沿着地图精确导航。 @@ -204,7 +204,7 @@ graphify trae-cn uninstall | graphify 源码 + Transformer 论文 | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | | httpx(合成 Python 库) | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | -Token 压缩效果会随着语料规模增大而更明显。6 个文件本来就塞得进上下文窗口,所以 graphify 在这种场景里的价值更多是结构清晰度,而不是 token 压缩。到了 52 个文件(代码 + 论文 + 图片)这种规模,就能做到 71x+。每个 `worked/` 目录里都带了原始输入和真实输出(`GRAPH_REPORT.md`、`graph.json`),你可以自己跑一遍核对数字。 +Token 压缩效果会随着语料规模增大而更明显。6 个文件本来就塞得进上下文窗口,所以 graphify 在这种场景里的价值更多是结构清晰度,而不是 token 压缩。到了 52 个文件(代码 + 论文 + 图片)这种规模,就能做到 71x+。每个 `worked/` 目录里都带了原始输入和真实输出(`GRAPH_REPORT.md`、`graph.helix`),你可以自己跑一遍核对数字。 ## 隐私 @@ -212,7 +212,7 @@ graphify 会把文档、论文和图片的内容发送给你所用 AI 编码助 ## 技术栈 -NetworkX + Leiden(graspologic)+ tree-sitter + vis.js。语义提取由 Claude(Claude Code)、GPT-4(Codex)或你当前平台所运行的模型完成。不需要 Neo4j,不需要 server,整体是纯本地运行。 +Helix + Leiden(native Leiden)+ tree-sitter + vis.js。语义提取由 Claude(Claude Code)、GPT-4(Codex)或你当前平台所运行的模型完成。不需要 Neo4j,不需要 server,整体是纯本地运行。
贡献 diff --git a/docs/translations/README.zh-TW.md b/docs/translations/README.zh-TW.md index 2bb90b4d4..0eae5fb12 100644 --- a/docs/translations/README.zh-TW.md +++ b/docs/translations/README.zh-TW.md @@ -27,13 +27,13 @@ graphify-out/ ├── graph.html 互動式圖譜 — 在任何瀏覽器中開啟 ├── GRAPH_REPORT.md 神級節點、令人驚訝的連接、建議問題 -├── graph.json 持久圖譜 — 幾週後仍可查詢 +├── graph.helix 持久圖譜 — 幾週後仍可查詢 └── cache/ SHA256 快取 — 重複執行只處理已變更的檔案 ``` ## 運作原理 -graphify 分三個階段工作。首先,確定性 AST 遍歷在不使用 LLM 的情況下從程式碼檔案中提取結構。然後使用 faster-whisper 在本地轉錄視訊和音訊檔案。最後,Claude 子代理並行處理文件、論文、圖片和轉錄文字。結果被合併到 NetworkX 圖譜中,使用 Leiden 進行聚類,並匯出為互動式 HTML、可查詢 JSON 和審計報告。 +graphify 分三個階段工作。首先,確定性 AST 遍歷在不使用 LLM 的情況下從程式碼檔案中提取結構。然後使用 faster-whisper 在本地轉錄視訊和音訊檔案。最後,Claude 子代理並行處理文件、論文、圖片和轉錄文字。結果被合併到 Helix 圖譜中,使用 Leiden 進行聚類,並匯出為互動式 HTML、可查詢 JSON 和審計報告。 每個關係都標記為 `EXTRACTED`、`INFERRED`(帶有置信度分數)或 `AMBIGUOUS`。 diff --git a/graphify/__init__.py b/graphify/__init__.py index 41d4b1f3d..b537b8928 100644 --- a/graphify/__init__.py +++ b/graphify/__init__.py @@ -6,7 +6,7 @@ def __getattr__(name): _map = { "extract": ("graphify.extract", "extract"), "collect_files": ("graphify.extract", "collect_files"), - "build_from_json": ("graphify.build", "build_from_json"), + "build_from_extraction": ("graphify.build", "build_from_extraction"), "cluster": ("graphify.cluster", "cluster"), "score_all": ("graphify.cluster", "score_all"), "cohesion_score": ("graphify.cluster", "cohesion_score"), @@ -14,7 +14,6 @@ def __getattr__(name): "surprising_connections": ("graphify.analyze", "surprising_connections"), "suggest_questions": ("graphify.analyze", "suggest_questions"), "generate": ("graphify.report", "generate"), - "to_json": ("graphify.export", "to_json"), "to_html": ("graphify.export", "to_html"), "to_svg": ("graphify.export", "to_svg"), "to_canvas": ("graphify.export", "to_canvas"), diff --git a/graphify/__main__.py b/graphify/__main__.py index d97d48fda..fcb1a0751 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -123,7 +123,7 @@ _StageTimer, _clone_repo, _default_graph_path, - _enforce_graph_size_cap_or_exit, + _validate_store_or_exit, _run_hook_guard, _SEARCH_NUDGE, _READ_NUDGE, @@ -510,24 +510,11 @@ def _run_cli() -> None: print(" install [--platform P] copy skill to platform config dir (claude|windows|codebuddy|codex|opencode|aider|amp|agents|claw|droid|trae|trae-cn|gemini|cursor|antigravity|hermes|kiro|pi|devin)") print(" uninstall remove graphify from all detected platforms in one shot") print(" --purge also delete graphify-out/ directory") - print(" path \"A\" \"B\" shortest path between two nodes in graph.json") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" path \"A\" \"B\" shortest path between two nodes in graph.helix") + print(" --store Helix store (default graphify-out/graph.helix)") print(" explain \"X\" plain-language explanation of a node and its neighbors") - print(" --graph path to graph.json (default graphify-out/graph.json)") - print(" diagnose multigraph report same-endpoint edge collapse risk in graph.json") - print(" --graph path to graph/extraction JSON") - print(" (default graphify-out/graph.json)") - print(" --json emit machine-readable JSON") - print(" --max-examples N max same-endpoint examples to print (default 5)") - print(" --directed force directed post-build simulation") - print(" --undirected force undirected post-build simulation") - print(" (default follows JSON directed flag;") - print(" raw extraction with no flag defaults directed)") - print(" --extract-path PATH extractor source for suppression scan") + print(" --store Helix store (default graphify-out/graph.helix)") print(" clone clone a GitHub repo locally and print its path for /graphify") - print(" merge-driver git merge driver: union-merge two graph.json files (set up via hook install)") - print(" merge-graphs merge two or more graph.json files into one cross-repo graph") - print(" --out output path (default: graphify-out/merged-graph.json)") print(" --branch checkout a specific branch (default: repo default)") print(" --out clone to a custom directory (default: ~/.graphify/repos//)") print(" add fetch a URL and save it to ./raw, then update the graph") @@ -536,12 +523,14 @@ def _run_cli() -> None: print(" --dir target directory (default: ./raw)") print(" watch watch a folder and rebuild the graph on code changes") print(" update re-extract code files and update the graph (no LLM needed)") - print(" --force overwrite graph.json even if the rebuild has fewer nodes") - print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") + print(" --retain-rollback retain one previous native generation") + print(" rollback [--store PATH] activate the explicitly retained previous generation") + print(" doctor [--deep] verify native counts or perform a full checksum audit") + print(" --force force a full source rescan") print(" --no-cluster skip clustering, write raw extraction only") - print(" cluster-only rerun clustering on an existing graph.json and regenerate report") + print(" cluster-only rerun clustering on an existing graph.helix generation") print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") - print(" --graph path to graph.json (default /graphify-out/graph.json)") + print(" --store Helix store (default /graphify-out/graph.helix)") print(" --no-label keep 'Community N' placeholders (skip LLM community naming)") print(" --backend= backend to use for community naming (default: auto-detect)") print(" --model= model to use for community naming") @@ -553,15 +542,15 @@ def _run_cli() -> None: print(" --model= model to use for community naming") print(" --max-concurrency=N parallel labeling LLM calls (default 4; forced to 1 for ollama/claude-cli)") print(" --batch-size=N communities per labeling LLM call (default 100)") - print(" query \"\" BFS traversal of graph.json for a question") + print(" query \"\" native traversal of graph.helix for a question") print(" --dfs use depth-first instead of breadth-first") print(" --context C explicit edge-context filter (repeatable)") print(" --budget N cap output at N tokens (default 2000)") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --store Helix store (default graphify-out/graph.helix)") print(" affected \"X\" reverse traversal to find nodes impacted by X") print(" --relation R edge relation to traverse in reverse (repeatable)") print(" --depth N reverse traversal depth (default 2)") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --store Helix store (default graphify-out/graph.helix)") print(" save-result save a Q&A result to graphify-out/memory/ for graph feedback loop") print(" --question Q the question asked") print(" --answer A the answer to save") @@ -575,14 +564,12 @@ def _run_cli() -> None: print(" reflect aggregate graphify-out/memory/ outcomes into a deterministic lessons doc") print(" --memory-dir DIR memory directory (default: graphify-out/memory)") print(" --out FILE output path (default: graphify-out/reflections/LESSONS.md)") - print(" --graph PATH graph.json, for community grouping + dropping stale nodes (optional)") - print(" --analysis PATH .graphify_analysis.json (optional, auto-detected next to --graph)") - print(" --labels PATH .graphify_labels.json (optional, auto-detected next to --graph)") + print(" --store PATH graph.helix store for community grouping (optional)") print(" --half-life-days N signal weight halves every N days (default 30)") print(" --min-corroboration N distinct useful results to prefer a node (default 2)") print(" check-update check needs_update flag and notify if semantic re-extraction is pending (cron-safe)") - print(" tree emit a D3 v7 collapsible-tree HTML for graph.json") - print(" --graph PATH path to graph.json (default graphify-out/graph.json)") + print(" tree emit a D3 v7 collapsible-tree HTML from graph.helix") + print(" --store PATH Helix store (default graphify-out/graph.helix)") print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") print(" --root PATH filesystem root for the hierarchy") print(" --max-children N cap children per node (default 200)") @@ -614,12 +601,12 @@ def _run_cli() -> None: print(" --cargo extract crate→crate deps from Cargo.toml") print(" --global also merge the resulting graph into the global graph") print(" --as repo tag for --global (default: target directory name)") - print(" global add add/update a project graph in the global graph (~/.graphify/global-graph.json)") + print(" global add add/update a project in ~/.graphify/global-graph.helix") + print(" --retain-rollback retain one previous global generation") print(" --as repo tag (default: parent directory name)") print(" global remove remove a repo's nodes from the global graph") print(" global list list repos in the global graph") - print(" global path print path to the global graph file") - print(" benchmark [graph.json] measure token reduction vs naive full-corpus approach") + print(" benchmark [graph.helix] measure token reduction vs naive full-corpus approach") print(" export callflow-html emit Mermaid-based architecture/call-flow HTML") print(" hook install install post-commit/post-checkout git hooks (all platforms)") print(" hook uninstall remove git hooks") diff --git a/graphify/_minhash.py b/graphify/_minhash.py index aabeb654f..c3a94cb43 100644 --- a/graphify/_minhash.py +++ b/graphify/_minhash.py @@ -44,7 +44,7 @@ def __init__(self, num_perm: int = 128) -> None: self._a, self._b = _mh_coeffs(num_perm) def update(self, v: bytes) -> None: - hv = np.uint64(struct.unpack(" str: - data = graph.nodes[node_id] +def _node_label(graph: Any, node_id: Any) -> str: + data = node_attributes(graph, node_id) return str(data.get("label") or node_id) @@ -56,8 +56,8 @@ def _normalize_label(label: str) -> str: def _prefer_file_node( - graph: nx.Graph, - node_ids: list[str], + graph: Any, + node_ids: list[Any], query: str, ) -> str | None: """Return the file-level node when a source_file query matches many nodes.""" @@ -65,8 +65,8 @@ def _prefer_file_node( exact_file_nodes = [ node_id for node_id in node_ids - if str(graph.nodes[node_id].get("source_location", "")) == "L1" - and _normalize_label(str(graph.nodes[node_id].get("label", ""))) == query_basename + if str(node_attributes(graph, node_id).get("source_location", "")) == "L1" + and _normalize_label(str(node_attributes(graph, node_id).get("label", ""))) == query_basename ] if len(exact_file_nodes) == 1: return exact_file_nodes[0] @@ -74,7 +74,7 @@ def _prefer_file_node( l1_nodes = [ node_id for node_id in node_ids - if str(graph.nodes[node_id].get("source_location", "")) == "L1" + if str(node_attributes(graph, node_id).get("source_location", "")) == "L1" ] if len(l1_nodes) == 1: return l1_nodes[0] @@ -82,7 +82,7 @@ def _prefer_file_node( basename_nodes = [ node_id for node_id in node_ids - if _normalize_label(str(graph.nodes[node_id].get("label", ""))) == query_basename + if _normalize_label(str(node_attributes(graph, node_id).get("label", ""))) == query_basename ] if len(basename_nodes) == 1: return basename_nodes[0] @@ -90,17 +90,22 @@ def _prefer_file_node( return None -def resolve_seed(graph: nx.Graph, query: str) -> str | None: +def resolve_seed(graph: Any, query: str, *, node_query: Any) -> Any | None: + """Resolve a seed from a bounded native predicate projection.""" # A trailing path separator must not change a source-file match — serve's # _find_node tokenizes the path (which drops it), so strip it here for parity # (otherwise `affected "src/x.ts/"` returned None while `explain` resolved it). query = query.rstrip("/\\") or query - if query in graph: + if graph.contains_node(query): return query + if node_query is None: + raise RuntimeError("affected analysis requires the native Helix query interface") query_lower = _normalize_label(query) + candidate_ids = node_query.candidate_ids([query]) exact_label_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) + node_id + for node_id in candidate_ids + if (data := node_attributes(graph, node_id)) is not None if _normalize_label(str(data.get("label", ""))) == query_lower ] if len(exact_label_matches) == 1: @@ -110,15 +115,17 @@ def resolve_seed(graph: nx.Graph, query: str) -> str | None: # contains pass. Match on the undecorated name before giving up. query_bare = _bare_name(query_lower) bare_name_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) + node_id + for node_id in candidate_ids + if (data := node_attributes(graph, node_id)) is not None if _bare_name(str(data.get("label", ""))) == query_bare ] if len(bare_name_matches) == 1: return bare_name_matches[0] exact_source_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) + node_id + for node_id in candidate_ids + if (data := node_attributes(graph, node_id)) is not None if _normalize_label(str(data.get("source_file", ""))) == query_lower ] if len(exact_source_matches) == 1: @@ -128,8 +135,9 @@ def resolve_seed(graph: nx.Graph, query: str) -> str | None: if preferred_file_node is not None: return preferred_file_node contains_matches = [ - str(node_id) - for node_id, data in graph.nodes(data=True) + node_id + for node_id in candidate_ids + if (data := node_attributes(graph, node_id)) is not None if query_lower in _normalize_label(str(data.get("label", ""))) ] if len(contains_matches) == 1: @@ -138,16 +146,15 @@ def resolve_seed(graph: nx.Graph, query: str) -> str | None: def affected_nodes( - graph: nx.Graph, - seed: str, + graph: Any, + seed: Any, *, relations: Iterable[str] = DEFAULT_AFFECTED_RELATIONS, depth: int = 2, ) -> list[AffectedHit]: - relation_set = set(relations) - seen = {seed} - queue: deque[tuple[str, int]] = deque([(seed, 0)]) - hits: list[AffectedHit] = [] + from helixdb.graph import TraversalOptions + + relation_list = tuple(dict.fromkeys(str(relation) for relation in relations)) # #1669: seed the reverse walk with the root's own member nodes (one outward # `method`/`contains` hop). A caller can bind to a class's method node rather @@ -156,56 +163,46 @@ def affected_nodes( # otherwise. The member nodes are seeds only (not reported as hits), and # `method`/`contains` stay out of the general relation-filtered walk, so this # adds no forward noise anywhere else. - if hasattr(graph, "out_edges"): - member_edges = graph.out_edges(seed, data=True) - else: - member_edges = ( - (s, t, d) for s, t, d in graph.edges(data=True) if s == seed - ) - for _s, member, data in member_edges: - if str(data.get("relation", "")) not in ("method", "contains"): + member_seeds: list[Any] = [] + member_edges = (graph.edge(edge_id) for edge_id in graph.out_edge_ids(seed)) + for edge in member_edges: + if edge is None: continue - member = str(member) - if member not in seen: - seen.add(member) - queue.append((member, 0)) - - while queue: - current, current_depth = queue.popleft() - if current_depth >= depth: + member, data = edge.target, edge_attributes(edge) + if str(data.get("relation", "")) not in ("method", "contains"): continue - if hasattr(graph, "in_edges"): - incoming = graph.in_edges(current, data=True) - else: - incoming = ( - (source, target, data) - for source, target, data in graph.edges(data=True) - if target == current - ) - for source, _target, data in incoming: - relation = str(data.get("relation", "")) - if relation not in relation_set: - continue - source = str(source) - if source in seen: - continue - seen.add(source) - hit = AffectedHit(source, current_depth + 1, relation) - hits.append(hit) - queue.append((source, current_depth + 1)) - - return hits + if member != seed and member not in member_seeds: + member_seeds.append(member) + + seeds = (seed, *member_seeds) + result = graph.traverse(TraversalOptions( + seeds=seeds, + max_depth=max(0, depth), + direction="in", + allowed_labels=relation_list, + )) + seed_set = set(seeds) + via_relation = { + traversed.source: str(traversed.label or "") + for traversed in result.discovery_edges + } + return [ + AffectedHit(visit.node_id, visit.depth, via_relation.get(visit.node_id, "")) + for visit in result.visits + if visit.node_id not in seed_set + ] def format_affected( - graph: nx.Graph, + graph: Any, query: str, *, + node_query: Any, relations: Iterable[str] = DEFAULT_AFFECTED_RELATIONS, depth: int = 2, ) -> str: relation_list = tuple(relations) - seed = resolve_seed(graph, query) + seed = resolve_seed(graph, query, node_query=node_query) if seed is None: return f"No unique node match for {query}" @@ -220,34 +217,19 @@ def format_affected( return "\n".join(lines) for hit in hits: - data = graph.nodes[hit.node_id] + data = node_attributes(graph, hit.node_id) lines.append( f"- {_node_label(graph, hit.node_id)} [{hit.via_relation}] {_format_location(data)}" ) return "\n".join(lines) -def load_graph(path: Path) -> nx.Graph: - import json - from networkx.readwrite import json_graph - +def load_graph(path: Path): + """Open the active immutable graph from a Helix store directory.""" try: - raw = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: + return load_helix_graph(path).graph + except (OSError, RuntimeError, ValueError) as exc: raise RuntimeError( - f"Cannot read graph file {path}: {exc}. " + f"Cannot open Helix store {path}: {exc}. " "Re-run 'graphify extract' to regenerate it." ) from exc - # Force directed so stored caller→callee direction survives the round-trip; - # mirrors serve.py and __main__.py (#1174). - raw = {**raw, "directed": True} - # Normalize the edge key: graphify's `extract` output uses "edges" while - # networkx's node_link_data default is "links". Without this, an edges-keyed - # graph.json raises an uncaught KeyError: 'links' here — every other loader - # (__main__.py) already normalizes this (#738; same class as #1198). - if "links" not in raw and "edges" in raw: - raw = dict(raw, links=raw["edges"]) - try: - return json_graph.node_link_graph(raw, edges="links") - except TypeError: - return json_graph.node_link_graph(raw) diff --git a/graphify/always_on/agents-md.md b/graphify/always_on/agents-md.md index 6511cd1dd..f3725dd93 100644 --- a/graphify/always_on/agents-md.md +++ b/graphify/always_on/agents-md.md @@ -5,7 +5,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community st When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. diff --git a/graphify/always_on/antigravity-rules.md b/graphify/always_on/antigravity-rules.md index 0fc786414..1b2f06f24 100644 --- a/graphify/always_on/antigravity-rules.md +++ b/graphify/always_on/antigravity-rules.md @@ -8,7 +8,7 @@ description: Consult the graphify knowledge graph at graphify-out/ for codebase This project has a graphify knowledge graph at graphify-out/. Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- For codebase or architecture questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. - If graphify-out/wiki/index.md exists, navigate it instead of reading raw files - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context - After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) diff --git a/graphify/always_on/claude-md.md b/graphify/always_on/claude-md.md index 417efeb27..f0b390010 100644 --- a/graphify/always_on/claude-md.md +++ b/graphify/always_on/claude-md.md @@ -3,7 +3,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. - After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/graphify/always_on/gemini-md.md b/graphify/always_on/gemini-md.md index 417efeb27..f0b390010 100644 --- a/graphify/always_on/gemini-md.md +++ b/graphify/always_on/gemini-md.md @@ -3,7 +3,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. - After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/graphify/always_on/kiro-steering.md b/graphify/always_on/kiro-steering.md index cb6f4543d..0e4221868 100644 --- a/graphify/always_on/kiro-steering.md +++ b/graphify/always_on/kiro-steering.md @@ -2,4 +2,4 @@ inclusion: always --- -graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. +graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. diff --git a/graphify/always_on/vscode-instructions.md b/graphify/always_on/vscode-instructions.md index 9cb983c95..aa7a32840 100644 --- a/graphify/always_on/vscode-instructions.md +++ b/graphify/always_on/vscode-instructions.md @@ -1,7 +1,7 @@ ## graphify For any question about this repo's architecture, structure, components, or how to add/modify/find -code, your first action should be `graphify query ""` when `graphify-out/graph.json` +code, your first action should be `graphify query ""` when `graphify-out/graph.helix` exists. Use `graphify path "" ""` for relationship questions and `graphify explain ""` for focused-concept questions. These return a scoped subgraph, usually much smaller than the full report or raw grep output. diff --git a/graphify/analyze.py b/graphify/analyze.py index cb6c76484..c41ff07a3 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -1,9 +1,23 @@ """Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions.""" from __future__ import annotations from pathlib import Path -import networkx as nx +from typing import Any -from graphify.build import edge_data +from graphify.helix.access import ( + degree as _degree, + degree_map, + edge_rows as _edges, + first_edge_attributes as edge_data, + node_attributes, + node_ids, +) +from graphify.helix.model import graphify_attributes + + +def _graphify_betweenness_options(): + from helixdb.graph import BetweennessOptions + + return BetweennessOptions(mode="auto", sample_count=100, seed=42, exact_through=1_000) # Builtin/mock names that can appear as annotation-derived nodes in pre-existing # graphs. Excluded from god-node ranking so they don't displace real abstractions @@ -52,7 +66,7 @@ def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]: return {n: cid for cid, nodes in communities.items() for n in nodes} -def _is_file_node(G: nx.Graph, node_id: str) -> bool: +def _is_file_node(G: Any, node_id: str) -> bool: """ Return True if this node is a file-level hub node (e.g. 'client', 'models') or an AST method stub (e.g. '.auth_flow()', '.__init__()'). @@ -60,7 +74,7 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: These are synthetic nodes created by the AST extractor and should be excluded from god nodes, surprising connections, and knowledge gap reporting. """ - attrs = G.nodes[node_id] + attrs = node_attributes(G, node_id) label = attrs.get("label", "") if not label: return False @@ -75,7 +89,7 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: return True # Module-level function stub: labeled 'function_name()' - only has a contains edge # These are real functions but structurally isolated by definition; not a gap worth flagging - if label.endswith("()") and G.degree(node_id) <= 1: + if label.endswith("()") and _degree(G, node_id) <= 1: return True return False @@ -88,8 +102,8 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: }) -def _is_json_key_node(G: nx.Graph, node_id: str) -> bool: - attrs = G.nodes[node_id] +def _is_json_key_node(G: Any, node_id: str) -> bool: + attrs = node_attributes(G, node_id) src = (attrs.get("source_file") or "").lower() if not src.endswith(".json"): return False @@ -97,23 +111,23 @@ def _is_json_key_node(G: nx.Graph, node_id: str) -> bool: return label in _JSON_NOISE_LABELS -def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: +def god_nodes(G: Any, top_n: int = 10) -> list[dict]: """Return the top_n most-connected real entities - the core abstractions. File-level hub nodes are excluded: they accumulate import/contains edges mechanically and don't represent meaningful architectural abstractions. """ - degree = dict(G.degree()) + degree = {row.node_id: int(row.degree) for row in G.degrees()} sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True) result = [] for node_id, deg in sorted_nodes: if _is_file_node(G, node_id) or _is_concept_node(G, node_id) or _is_json_key_node(G, node_id): continue - if G.nodes[node_id].get("label", "") in _BUILTIN_NOISE_LABELS: + if node_attributes(G, node_id).get("label", "") in _BUILTIN_NOISE_LABELS: continue result.append({ "id": node_id, - "label": G.nodes[node_id].get("label", node_id), + "label": node_attributes(G, node_id).get("label", node_id), "degree": deg, }) if len(result) >= top_n: @@ -122,7 +136,7 @@ def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: def surprising_connections( - G: nx.Graph, + G: Any, communities: dict[int, list[str]] | None = None, top_n: int = 5, ) -> list[dict]: @@ -142,7 +156,8 @@ def surprising_connections( # Identify unique source files (ignore empty/null source_file) source_files = { data.get("source_file", "") - for _, data in G.nodes(data=True) + for node in G.nodes() + if (data := graphify_attributes(node.attributes)) is not None if data.get("source_file", "") } is_multi_source = len(source_files) > 1 @@ -153,7 +168,7 @@ def surprising_connections( return _cross_community_surprises(G, communities or {}, top_n) -def _is_concept_node(G: nx.Graph, node_id: str) -> bool: +def _is_concept_node(G: Any, node_id: str) -> bool: """ Return True if this node is a manually-injected semantic concept node rather than a real entity found in source code. @@ -162,7 +177,7 @@ def _is_concept_node(G: nx.Graph, node_id: str) -> bool: - Empty source_file - source_file doesn't look like a real file path (no extension) """ - data = G.nodes[node_id] + data = node_attributes(G, node_id) source = data.get("source_file", "") if not source: return True @@ -192,7 +207,7 @@ def _top_level_dir(path: str) -> str: def _surprise_score( - G: nx.Graph, + G: Any, u: str, v: str, data: dict, @@ -241,7 +256,7 @@ def _surprise_score( score += 2 reasons.append("connects across different repos/directories") - # 4. Cross-community bonus - Leiden says these are structurally distant + # 4. Cross-community bonus - Leiden says these are structurally distant cid_u = node_community.get(u) cid_v = node_community.get(v) if cid_u is not None and cid_v is not None and cid_u != cid_v and not _suppress_structural: @@ -254,18 +269,18 @@ def _surprise_score( reasons.append("semantically similar concepts with no structural link") # 5. Peripheral→hub: a low-degree node connecting to a high-degree one - deg_u = degrees[u] if degrees is not None else G.degree(u) - deg_v = degrees[v] if degrees is not None else G.degree(v) + deg_u = degrees[u] if degrees is not None else _degree(G, u) + deg_v = degrees[v] if degrees is not None else _degree(G, v) if min(deg_u, deg_v) <= 2 and max(deg_u, deg_v) >= 5: score += 1 - peripheral = G.nodes[u].get("label", u) if deg_u <= 2 else G.nodes[v].get("label", v) - hub = G.nodes[v].get("label", v) if deg_u <= 2 else G.nodes[u].get("label", u) + peripheral = node_attributes(G, u).get("label", u) if deg_u <= 2 else node_attributes(G, v).get("label", v) + hub = node_attributes(G, v).get("label", v) if deg_u <= 2 else node_attributes(G, u).get("label", u) reasons.append(f"peripheral node `{peripheral}` unexpectedly reaches hub `{hub}`") return score, reasons -def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: int) -> list[dict]: +def _cross_file_surprises(G: Any, communities: dict[int, list[str]], top_n: int) -> list[dict]: """ Cross-file edges between real code/doc entities, ranked by a composite surprise score rather than confidence alone. @@ -280,10 +295,10 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: Each result includes a 'why' field explaining what makes it non-obvious. """ node_community = _node_community_map(communities) - degrees = dict(G.degree()) + degrees = degree_map(G) candidates = [] - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edges(G): relation = data.get("relation", "") if relation in ("imports", "imports_from", "contains", "method"): continue @@ -292,26 +307,26 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: if _is_file_node(G, u) or _is_file_node(G, v): continue - u_source = G.nodes[u].get("source_file", "") - v_source = G.nodes[v].get("source_file", "") + u_source = node_attributes(G, u).get("source_file", "") + v_source = node_attributes(G, v).get("source_file", "") if not u_source or not v_source or u_source == v_source: continue score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source, degrees) src_id = data.get("_src", u) - if src_id not in G.nodes: + if not G.contains_node(src_id): src_id = u tgt_id = data.get("_tgt", v) - if tgt_id not in G.nodes: + if not G.contains_node(tgt_id): tgt_id = v candidates.append({ "_score": score, - "source": G.nodes[src_id].get("label", src_id), - "target": G.nodes[tgt_id].get("label", tgt_id), + "source": node_attributes(G, src_id).get("label", src_id), + "target": node_attributes(G, tgt_id).get("label", tgt_id), "source_files": [ - G.nodes[src_id].get("source_file", ""), - G.nodes[tgt_id].get("source_file", ""), + node_attributes(G, src_id).get("source_file", ""), + node_attributes(G, tgt_id).get("source_file", ""), ], "confidence": data.get("confidence", "EXTRACTED"), "relation": relation, @@ -329,7 +344,7 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: def _cross_community_surprises( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], top_n: int, ) -> list[dict]: @@ -342,21 +357,28 @@ def _cross_community_surprises( """ if not communities: # No community info - use edge betweenness centrality - if G.number_of_edges() == 0: + if G.edge_count == 0: return [] - if G.number_of_nodes() > 5000: + if G.node_count > 5000: return [] - betweenness = nx.edge_betweenness_centrality(G) - top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n] + top_edges = sorted( + G.edge_betweenness_centrality(_graphify_betweenness_options()), + key=lambda row: row.score, + reverse=True, + )[:top_n] result = [] - for (u, v), score in top_edges: + for row in top_edges: + edge = G.edge(row.edge_id) + if edge is None: + continue + u, v, score = edge.source, edge.target, row.score data = edge_data(G, u, v) result.append({ - "source": G.nodes[u].get("label", u), - "target": G.nodes[v].get("label", v), + "source": node_attributes(G, u).get("label", u), + "target": node_attributes(G, v).get("label", v), "source_files": [ - G.nodes[u].get("source_file", ""), - G.nodes[v].get("source_file", ""), + node_attributes(G, u).get("source_file", ""), + node_attributes(G, v).get("source_file", ""), ], "confidence": data.get("confidence", "EXTRACTED"), "relation": data.get("relation", ""), @@ -368,7 +390,7 @@ def _cross_community_surprises( node_community = _node_community_map(communities) surprises = [] - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edges(G): cid_u = node_community.get(u) cid_v = node_community.get(v) if cid_u is None or cid_v is None or cid_u == cid_v: @@ -382,17 +404,17 @@ def _cross_community_surprises( # This edge crosses community boundaries - interesting confidence = data.get("confidence", "EXTRACTED") src_id = data.get("_src", u) - if src_id not in G.nodes: + if not G.contains_node(src_id): src_id = u tgt_id = data.get("_tgt", v) - if tgt_id not in G.nodes: + if not G.contains_node(tgt_id): tgt_id = v surprises.append({ - "source": G.nodes[src_id].get("label", src_id), - "target": G.nodes[tgt_id].get("label", tgt_id), + "source": node_attributes(G, src_id).get("label", src_id), + "target": node_attributes(G, tgt_id).get("label", tgt_id), "source_files": [ - G.nodes[src_id].get("source_file", ""), - G.nodes[tgt_id].get("source_file", ""), + node_attributes(G, src_id).get("source_file", ""), + node_attributes(G, tgt_id).get("source_file", ""), ], "confidence": confidence, "relation": relation, @@ -417,7 +439,7 @@ def _cross_community_surprises( def suggest_questions( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], community_labels: dict[int, str], top_n: int = 7, @@ -434,10 +456,10 @@ def suggest_questions( node_community = _node_community_map(communities) # 1. AMBIGUOUS edges → unresolved relationship questions - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edges(G): if data.get("confidence") == "AMBIGUOUS": - ul = G.nodes[u].get("label", u) - vl = G.nodes[v].get("label", v) + ul = node_attributes(G, u).get("label", u) + vl = node_attributes(G, v).get("label", v) relation = data.get("relation", "related to") questions.append({ "type": "ambiguous_edge", @@ -446,9 +468,11 @@ def suggest_questions( }) # 2. Bridge nodes (high betweenness) → cross-cutting concern questions - if G.number_of_edges() > 0: - k = min(100, G.number_of_nodes()) if G.number_of_nodes() > 1000 else None - betweenness = nx.betweenness_centrality(G, k=k, seed=42) + if G.edge_count > 0: + betweenness = { + row.node_id: row.score + for row in G.betweenness_centrality(_graphify_betweenness_options()) + } # Top bridge nodes that are NOT file-level hubs bridges = sorted( [(n, s) for n, s in betweenness.items() @@ -457,11 +481,15 @@ def suggest_questions( reverse=True, )[:3] for node_id, score in bridges: - label = G.nodes[node_id].get("label", node_id) + label = node_attributes(G, node_id).get("label", node_id) cid = node_community.get(node_id) comm_label = community_labels.get(cid, f"Community {cid}") if cid is not None else "unknown" neighbors = list(G.neighbors(node_id)) - neighbor_comms = {node_community.get(n) for n in neighbors if node_community.get(n) != cid} + neighbor_comms = { + neighbor_cid + for n in neighbors + if (neighbor_cid := node_community.get(n)) is not None and neighbor_cid != cid + } if neighbor_comms: other_labels = [community_labels.get(c, f"Community {c}") for c in neighbor_comms] questions.append({ @@ -471,7 +499,7 @@ def suggest_questions( }) # 3. God nodes with many INFERRED edges → verification questions - degree = dict(G.degree()) + degree = {row.node_id: int(row.degree) for row in G.degrees()} top_nodes = sorted( [(n, d) for n, d in degree.items() if not _is_file_node(G, n)], key=lambda x: x[1], @@ -479,22 +507,22 @@ def suggest_questions( )[:5] for node_id, _ in top_nodes: inferred = [ - (u, v, d) for u, v, d in G.edges(node_id, data=True) + (u, v, d) for u, v, d, _ in _edges(G, node_id) if d.get("confidence") == "INFERRED" ] if len(inferred) >= 2: - label = G.nodes[node_id].get("label", node_id) + label = node_attributes(G, node_id).get("label", node_id) # Use _src/_tgt to get the correct direction; fall back to v (the other node) others = [] for u, v, d in inferred[:2]: src_id = d.get("_src", u) - if src_id not in G.nodes: + if not G.contains_node(src_id): src_id = u tgt_id = d.get("_tgt", v) - if tgt_id not in G.nodes: + if not G.contains_node(tgt_id): tgt_id = v other_id = tgt_id if src_id == node_id else src_id - others.append(G.nodes[other_id].get("label", other_id)) + others.append(node_attributes(G, other_id).get("label", other_id)) questions.append({ "type": "verify_inferred", "question": f"Are the {len(inferred)} inferred relationships involving `{label}` (e.g. with `{others[0]}` and `{others[1]}`) actually correct?", @@ -503,14 +531,14 @@ def suggest_questions( # 4. Isolated or weakly-connected nodes → exploration questions isolated = [ - n for n in G.nodes() - if G.degree(n) <= 1 + n for n in node_ids(G) + if _degree(G, n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n) - and G.nodes[n].get("file_type") != "rationale" + and node_attributes(G, n).get("file_type") != "rationale" ] if isolated: - labels = [G.nodes[n].get("label", n) for n in isolated[:3]] + labels = [node_attributes(G, n).get("label", n) for n in isolated[:3]] questions.append({ "type": "isolated_nodes", "question": f"What connects {', '.join(f'`{l}`' for l in labels)} to the rest of the system?", @@ -544,7 +572,7 @@ def suggest_questions( return questions[:top_n] -def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict: +def graph_diff(G_old: Any, G_new: Any) -> dict: """Compare two graph snapshots and return what changed. Returns: @@ -556,40 +584,40 @@ def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict: "summary": "3 new nodes, 5 new edges, 1 node removed" } """ - old_nodes = set(G_old.nodes()) - new_nodes = set(G_new.nodes()) + old_nodes = {node.id for node in G_old.nodes()} + new_nodes = {node.id for node in G_new.nodes()} added_node_ids = new_nodes - old_nodes removed_node_ids = old_nodes - new_nodes new_nodes_list = [ - {"id": n, "label": G_new.nodes[n].get("label", n)} + {"id": n, "label": node_attributes(G_new, n).get("label", n)} for n in added_node_ids ] removed_nodes_list = [ - {"id": n, "label": G_old.nodes[n].get("label", n)} + {"id": n, "label": node_attributes(G_old, n).get("label", n)} for n in removed_node_ids ] - def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple: - if G.is_directed(): + def edge_key(G: Any, u: str, v: str, data: dict) -> tuple: + if G.directed: return (u, v, data.get("relation", "")) return (min(u, v), max(u, v), data.get("relation", "")) old_edge_keys = { edge_key(G_old, u, v, d) - for u, v, d in G_old.edges(data=True) + for u, v, d, _ in _edges(G_old) } new_edge_keys = { edge_key(G_new, u, v, d) - for u, v, d in G_new.edges(data=True) + for u, v, d, _ in _edges(G_new) } added_edge_keys = new_edge_keys - old_edge_keys removed_edge_keys = old_edge_keys - new_edge_keys new_edges_list = [] - for u, v, d in G_new.edges(data=True): + for u, v, d, _ in _edges(G_new): if edge_key(G_new, u, v, d) in added_edge_keys: new_edges_list.append({ "source": u, @@ -599,7 +627,7 @@ def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple: }) removed_edges_list = [] - for u, v, d in G_old.edges(data=True): + for u, v, d, _ in _edges(G_old): if edge_key(G_old, u, v, d) in removed_edge_keys: removed_edges_list.append({ "source": u, @@ -629,7 +657,7 @@ def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple: def find_import_cycles( - G: nx.Graph, + G: Any, max_cycle_length: int = 5, top_n: int = 20, ) -> list[dict]: @@ -653,15 +681,17 @@ def find_import_cycles( } """ def _endpoint_source_file(node_id: str) -> str: - attrs = G.nodes.get(node_id, {}) + if not G.contains_node(node_id): + return "" + attrs = node_attributes(G, node_id) src_file = attrs.get("source_file", "") return src_file if isinstance(src_file, str) else "" # Step 1: Build a directed file-level graph from import/re-export edges. # IMPORTANT: resolve endpoints using source_file only; never infer from label/id. - file_graph = nx.DiGraph() + adjacency: dict[str, set[str]] = {} - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edges(G): rel = data.get("relation", "") if rel not in ("imports_from", "re_exports"): continue @@ -693,21 +723,27 @@ def _endpoint_source_file(node_id: str) -> str: if not tgt_file: continue - file_graph.add_edge(src_file_attr, tgt_file) + adjacency.setdefault(src_file_attr, set()).add(tgt_file) - if not file_graph.edges(): + if not adjacency: return [] # Step 2: Find simple cycles, bounded by length. - # Pass length_bound so networkx prunes during enumeration rather than - # enumerating all elementary cycles and post-filtering — avoids exponential - # blowup on dense graphs with many long cycles (#1196). + # Bound enumeration to avoid exponential blowup on dense graphs. cycles: list[list[str]] = [] - for cycle in nx.simple_cycles(file_graph, length_bound=max_cycle_length): - if len(cycle) <= max_cycle_length: - cycles.append(cycle) + + def walk(start: str, current: str, path: list[str], seen: set[str]) -> None: + if len(cycles) >= top_n * 10 or len(path) > max_cycle_length: + return + for target in sorted(adjacency.get(current, ())): + if target == start: + cycles.append(path.copy()) + elif target not in seen and len(path) < max_cycle_length: + walk(start, target, [*path, target], {*seen, target}) + + for start in sorted(adjacency): + walk(start, start, [start], {start}) if len(cycles) >= top_n * 10: - # Stop early to avoid combinatorial explosion break # Step 3: Sort by length (shortest = tightest coupling), then deduplicate. diff --git a/graphify/benchmark.py b/graphify/benchmark.py index ee9c3925e..7f07d02aa 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -1,14 +1,12 @@ """Token-reduction benchmark - measures how much context graphify saves vs naive full-corpus approach.""" from __future__ import annotations -import json import sys -from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph +from typing import Any from graphify.build import edge_data +from graphify.helix.model import graphify_attributes, node_attributes +from graphify.helix.persistence import DEFAULT_PROJECT_STORE, load_graph from graphify.serve import _query_terms -from graphify.paths import default_graph_json as _default_graph_json _CHARS_PER_TOKEN = 4 # standard approximation @@ -37,11 +35,12 @@ def _estimate_tokens(text: str) -> int: return max(1, len(text) // _CHARS_PER_TOKEN) -def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int: +def _query_subgraph_tokens(G: Any, question: str, depth: int = 3) -> int: """Run BFS from best-matching nodes and return estimated tokens in the subgraph context.""" terms = _query_terms(question) scored = [] - for nid, data in G.nodes(data=True): + for node in G.nodes(): + nid, data = node.id, graphify_attributes(node.attributes) label = data.get("label", "").lower() score = sum(1 for t in terms if t in label) if score > 0: @@ -51,27 +50,22 @@ def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int: if not start_nodes: return 0 - visited: set[str] = set(start_nodes) - frontier = set(start_nodes) - edges_seen: list[tuple] = [] - for _ in range(depth): - next_frontier: set[str] = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in visited: - next_frontier.add(neighbor) - edges_seen.append((n, neighbor)) - visited.update(next_frontier) - frontier = next_frontier + from helixdb.graph import TraversalOptions + + traversal = G.traverse(TraversalOptions( + seeds=tuple(start_nodes), max_depth=depth, direction="both" + )) + visited = {visit.node_id for visit in traversal.visits} lines = [] for nid in visited: - d = G.nodes[nid] + d = node_attributes(G, nid) lines.append(f"NODE {d.get('label', nid)} src={d.get('source_file', '')} loc={d.get('source_location', '')}") - for u, v in edges_seen: + for traversed in traversal.discovery_edges: + u, v = traversed.source, traversed.target if u in visited and v in visited: d = edge_data(G, u, v) - lines.append(f"EDGE {G.nodes[u].get('label', u)} --{d.get('relation', '')}--> {G.nodes[v].get('label', v)}") + lines.append(f"EDGE {node_attributes(G, u).get('label', u)} --{d.get('relation', '')}--> {node_attributes(G, v).get('label', v)}") return _estimate_tokens("\n".join(lines)) @@ -99,19 +93,13 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ - graph_path = graph_path or _default_graph_json() - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(Path(graph_path)) - data = json.loads(Path(graph_path).read_text(encoding="utf-8")) - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) + G = load_graph(graph_path or DEFAULT_PROJECT_STORE).graph if corpus_words is None: # Rough estimate: each node label is ~3 words, plus source context - corpus_words = G.number_of_nodes() * 50 + corpus_words = G.node_count * 50 + assert corpus_words is not None corpus_tokens = corpus_words * 100 // 75 # words → tokens (100 words ≈ 133 tokens) qs = questions or _SAMPLE_QUESTIONS @@ -130,8 +118,8 @@ def run_benchmark( return { "corpus_tokens": corpus_tokens, "corpus_words": corpus_words, - "nodes": G.number_of_nodes(), - "edges": G.number_of_edges(), + "nodes": G.node_count, + "edges": G.edge_count, "avg_query_tokens": avg_query_tokens, "reduction_ratio": reduction_ratio, "per_question": per_question, diff --git a/graphify/build.py b/graphify/build.py index 6992dbc57..af2786fa8 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -1,4 +1,4 @@ -# assemble node+edge dicts into a NetworkX graph, preserving edge direction +# Assemble extraction records into Helix build data, preserving edge direction. # # Node deduplication — three layers: # @@ -6,9 +6,8 @@ # emitted at most once per file, so duplicate class/function definitions in # the same source file are collapsed to the first occurrence. # -# 2. Between files (build): NetworkX G.add_node() is idempotent — calling it -# twice with the same ID overwrites the attributes with the second call's -# values. Nodes are added in extraction order (AST first, then semantic), +# 2. Between files (build): the transient Helix build DTO is last-writer-wins +# for duplicate IDs. Nodes are processed in extraction order (AST first, then semantic), # so if the same entity is extracted by both passes the semantic node # silently overwrites the AST node. This is intentional: semantic nodes # carry richer labels and cross-file context, while AST nodes have precise @@ -28,9 +27,15 @@ import sys import unicodedata from pathlib import Path -import networkx as nx +from typing import Any from .ids import make_id, normalize_id as _normalize_id -from .paths import default_graph_json as _default_graph_json +from .helix.access import all_edge_attributes, first_edge_attributes +from .helix.model import ( + EdgeData, + GraphBuildData, + NodeData, +) +from .helix.persistence import DEFAULT_PROJECT_STORE from .validate import validate_extraction @@ -75,7 +80,7 @@ # Hyperedge member lists are canonically keyed `nodes` (see graphify/llm.py -# extraction spec), but LLM/subagent drift and externally-supplied graph.json +# extraction spec), but LLM/subagent drift and externally supplied payloads # sometimes emit `members` or `node_ids`. _normalize_hyperedge_members folds # those aliases into `nodes` at ingest so every downstream consumer reads one # canonical key — mirroring the `from`/`to` edge-endpoint tolerance below. @@ -157,8 +162,8 @@ def _infer_merge_root(graph_path: Path) -> str | None: Prefers the committed ``graphify-out/.graphify_root`` marker — the authoritative scan root graphify records at build/watch time (#686/#1423) — then falls back to - the directory that contains the output dir (``graph.json``'s grandparent, i.e. - ``/graphify-out/graph.json`` -> ````). Returns None if neither + the directory that contains the output dir (the store's grandparent, i.e. + ``/graphify-out/graph.helix`` -> ````). Returns None if neither resolves, in which case normalization is a no-op (prior behavior). """ try: @@ -175,31 +180,20 @@ def _infer_merge_root(graph_path: Path) -> str | None: return None -def edge_data(G: nx.Graph, u: str, v: str) -> dict: - """Return one edge attribute dict for (u, v), tolerating MultiGraph. - - For MultiGraph/MultiDiGraph there can be multiple parallel edges; - this returns the first one (sufficient for callers that only need - relation/confidence for rendering). Fixes #796. - """ - raw = G[u][v] - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - return next(iter(raw.values()), {}) - return raw +def edge_data(G, u: str, v: str) -> dict: + """Return one native Helix edge attribute projection for ``(u, v)``.""" + return first_edge_attributes(G, u, v) -def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]: +def edge_datas(G, u: str, v: str) -> list[dict]: """Return every edge attribute dict for (u, v); always a list.""" - raw = G[u][v] - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - return list(raw.values()) - return [raw] + return all_edge_attributes(G, u, v) def dedupe_nodes(nodes: list[dict]) -> list[dict]: """Collapse nodes sharing an ``id``, last-writer-wins on attributes. - Mirrors what ``build_from_json``'s ``G.add_node`` does implicitly (idempotent; + Mirrors what ``build_from_extraction``'s ``G.add_node`` does implicitly (idempotent; a later node overwrites an earlier one's attributes). The ``--no-cluster`` write path dumps the raw node list without building a graph, so same-id nodes — e.g. a Swift ``type=module`` anchor emitted once per importing file (#1327) @@ -219,13 +213,9 @@ def dedupe_edges(edges: list[dict]) -> list[dict]: """Collapse exact parallel edges by ``(source, target, relation)``, keeping the first occurrence. - The clustered build path runs edges through a NetworkX ``DiGraph``, which - collapses parallel edges automatically. The ``--no-cluster`` and incremental - ``update`` write paths bypass NetworkX and concatenate edge lists raw, so - duplicates accumulate and edge counts become non-deterministic across build - modes / repeated updates (#1317). Deduping on the connectivity identity is - zero-signal-loss and restores idempotency. Callers that intentionally keep - parallel edges (multigraph output) must not use this. + Raw extraction and incremental updates can concatenate edge lists, so exact + duplicates would otherwise accumulate across repeated builds. Callers that + intentionally keep parallel edges (multigraph output) must not use this. """ seen: set[tuple] = set() out: list[dict] = [] @@ -238,6 +228,127 @@ def dedupe_edges(edges: list[dict]) -> list[dict]: return out +def build_unclustered_extraction( + extraction: dict, + *, + root: str | Path | None = None, +) -> GraphBuildData: + """Preserve the historical ``--no-cluster`` multigraph semantics. + + The v8 fast path wrote the deduplicated extraction records directly and + consumers rehydrated them as an undirected multigraph. Rehydration also + created empty external/stdlib endpoint nodes for otherwise dangling import + edges. Building a simple graph here silently collapsed relation-distinct + parallel edges and discarded those endpoints, so keep the raw topology in + the transient DTO and let Helix persist it natively. + """ + build_root = str(Path(root).resolve()) if root else None + raw_nodes = dedupe_nodes(list(extraction.get("nodes", []))) + raw_edges = dedupe_edges( + list(extraction.get("edges", extraction.get("links", []))) + ) + + node_attrs: dict[object, dict[str, Any]] = {} + for raw in raw_nodes: + if not isinstance(raw, dict) or "id" not in raw: + continue + node_id = raw["id"] + try: + hash(node_id) + except TypeError: + continue + attrs = {key: value for key, value in raw.items() if key != "id"} + if "source_file" in attrs: + attrs["source_file"] = _norm_source_file( + attrs["source_file"], build_root + ) + node_attrs[node_id] = attrs + + # Match the historical v8 multigraph loader's automatic per-pair integer keys. Explicit + # keys overwrite the same pair/key record, while unkeyed records receive + # the next unused integer for that undirected pair. + used_keys: dict[frozenset[object], set[object]] = {} + edge_by_identity: dict[tuple[frozenset[object], object], EdgeData] = {} + for raw in raw_edges: + if not isinstance(raw, dict): + continue + source = raw.get("source", raw.get("from")) + target = raw.get("target", raw.get("to")) + try: + hash(source) + hash(target) + except TypeError: + continue + if source is None or target is None: + continue + node_attrs.setdefault(source, {}) + node_attrs.setdefault(target, {}) + pair = frozenset((source, target)) + pair_keys = used_keys.setdefault(pair, set()) + if "key" in raw: + key = raw["key"] + try: + hash(key) + except TypeError: + continue + else: + key = len(pair_keys) + while key in pair_keys: + key += 1 + pair_keys.add(key) + attrs = { + attr: value + for attr, value in raw.items() + if attr + not in {"source", "target", "from", "to", "key", "target_file"} + } + relation = attrs.get("relation") + if not isinstance(relation, str) or not relation: + attrs["relation"] = "related_to" + if "source_file" in attrs: + attrs["source_file"] = _norm_source_file( + attrs["source_file"], build_root + ) + edge_by_identity[(pair, key)] = EdgeData(source, target, attrs, key) + + raw_hyperedges = extraction.get("hyperedges", []) + hyperedges = [] + if isinstance(raw_hyperedges, list): + for raw in raw_hyperedges: + if not isinstance(raw, dict): + hyperedges.append(raw) + continue + record = dict(raw) + if "source_file" in record: + record["source_file"] = _norm_source_file( + record["source_file"], build_root + ) + hyperedges.append(record) + graph_attributes = ( + {"hyperedges": list(hyperedges)} + if hyperedges + else {} + ) + reserved = { + "directed", + "multigraph", + "graph", + "nodes", + "edges", + "links", + "hyperedges", + } + return GraphBuildData( + kind="multigraph", + nodes=[NodeData(node_id, attrs) for node_id, attrs in node_attrs.items()], + edges=list(edge_by_identity.values()), + attributes=graph_attributes, + extras={ + key: value for key, value in extraction.items() if key not in reserved + }, + ) + + def _old_file_stems(rel: Path) -> list[str]: """Pre-migration stem forms a semantic fragment may have used for ``rel``. @@ -391,8 +502,14 @@ def _doc_twin_remap(nodes: list) -> dict[str, str]: return remap -def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: - """Build a NetworkX graph from an extraction dict. +def build_from_extraction( + extraction: dict, + *, + directed: bool = False, + multigraph: bool | None = None, + root: str | Path | None = None, +) -> GraphBuildData: + """Build transient Helix records from an extraction dict. directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. @@ -400,7 +517,10 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat relative to root so all nodes share a consistent path key (#932). """ _root = str(Path(root).resolve()) if root else None - # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. + directed = directed or bool(extraction.get("directed", False)) + if multigraph is None: + multigraph = bool(extraction.get("multigraph", False)) + # Accept the historical node-link spelling while normalizing in memory. if "edges" not in extraction and "links" in extraction: extraction = dict(extraction, edges=extraction["links"]) @@ -422,7 +542,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat file=sys.stderr, ) node["source_file"] = node.pop("source") - # Default missing/None file_type to "concept" so legacy graph.json + # Default missing/None file_type to "concept" so legacy native graph # entries (and stub nodes preserved by `_rebuild_code` from older # graphify versions that didn't always populate file_type) don't # trigger spurious "invalid file_type 'None'" validator warnings (#660). @@ -435,14 +555,13 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Canonicalize hyperedge member lists (#1561): producers sometimes key the # member list `members`/`node_ids` instead of `nodes`. Fold aliases onto # `nodes` here — BEFORE validation and the semantic-rekey loop below — so - # every downstream consumer (rekey, source_file relativize, to_json) reads + # every downstream consumer reads # one canonical key, the same way edge endpoints alias from/to at build. for he in extraction.get("hyperedges", []) or []: _normalize_hyperedge_members(he) errors = validate_extraction(extraction) - # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors. - real_errors = [e for e in errors if "does not match any node id" not in e] + real_errors = [error for error in errors if "does not match any node id" not in error] if real_errors: print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr) # Deterministic semantic re-key (#1504/#1509): the node-ID stem is now the @@ -498,11 +617,11 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if isinstance(he, dict) and isinstance(he.get("nodes"), list): he["nodes"] = [_doc_remap.get(n, n) for n in he["nodes"]] - G: nx.Graph = nx.DiGraph() if directed else nx.Graph() + node_attrs: dict[object, dict] = {} for node in extraction.get("nodes", []): # Skip dict nodes with a missing or non-hashable id (e.g. a list emitted - # by a buggy LLM extraction) so NetworkX add_node never raises - # TypeError: unhashable type. Non-dict nodes are deliberately left to + # by a buggy LLM extraction) so construction never raises a late + # TypeError. Non-dict nodes are deliberately left to # raise as before, so callers that probe build for shape errors (e.g. # the multigraph diagnostic) still observe the malformed shape. if isinstance(node, dict): @@ -519,8 +638,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat continue if "source_file" in node: node["source_file"] = _norm_source_file(node["source_file"], _root) - G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) - node_set = set(G.nodes()) + node_attrs[node["id"]] = {k: v for k, v in node.items() if k != "id"} + node_set = set(node_attrs) # #1145 (extended): merge LLM ghost-duplicate nodes into AST canonical nodes. # Original bug: AST uses parent-qualified IDs (mingpt_bpe_get_pairs) while LLM @@ -543,8 +662,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # canonical winner and the ambiguity decisions below don't flip run-to-run # with CPython's per-process string-hash seed (#1753) — the same reason the # edge-iteration loop further down sorts on purpose. - for nid in sorted(node_set): - attrs = G.nodes[nid] + for nid in sorted(node_set, key=repr): + attrs = node_attrs[nid] label = str(attrs.get("label", "")).strip() sf = str(attrs.get("source_file", "")) basename = Path(sf).name if sf else "" @@ -555,7 +674,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat key = (basename, label) if is_ast: # Two AST nodes on the same key is an ambiguous collision. - if key in _loc_nodes and G.nodes[_loc_nodes[key]].get("_origin") == "ast": + if key in _loc_nodes and node_attrs[_loc_nodes[key]].get("_origin") == "ast": _loc_collisions.add(key) # AST-origin nodes always overwrite a prior non-AST entry. _loc_nodes[key] = nid @@ -564,8 +683,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if existing is None: _loc_nodes[key] = nid elif ( - G.nodes[existing].get("_origin") != "ast" - and str(G.nodes[existing].get("source_file", "")) != sf + node_attrs[existing].get("_origin") != "ast" + and str(node_attrs[existing].get("source_file", "")) != sf ): # Two NON-AST nodes sharing (basename, label) but coming from # DIFFERENT files are distinct concepts (e.g. a same-named @@ -579,8 +698,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat _loc_collisions.add(key) # Pass 2: find ghosts — non-AST nodes that have an AST canonical twin. - for nid in sorted(node_set): - attrs = G.nodes[nid] + for nid in sorted(node_set, key=repr): + attrs = node_attrs[nid] if attrs.get("_origin") == "ast": continue # AST nodes are never ghosts label = str(attrs.get("label", "")).strip() @@ -601,13 +720,15 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat _ghost_remap[sem_id] = ast_id # Remove ghost nodes from the graph; edges will be re-pointed via norm_to_id. for ghost_id in _ghost_remap: - G.remove_node(ghost_id) + node_attrs.pop(ghost_id, None) node_set.discard(ghost_id) # Normalized ID map: lets edges survive when the LLM generates IDs with # slightly different casing or punctuation than the AST extractor. # e.g. "Session_ValidateToken" maps to "session_validatetoken". - norm_to_id: dict[str, str] = {_normalize_id(nid): nid for nid in node_set} + norm_to_id: dict[str, object] = { + _normalize_id(nid): nid for nid in node_set if isinstance(nid, str) + } # Also map ghost IDs to their canonical AST replacements. for ghost_id, canonical_id in _ghost_remap.items(): norm_to_id[_normalize_id(ghost_id)] = canonical_id @@ -644,7 +765,9 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat from graphify.extractors.base import _file_stem as _fs _alias_candidates: dict[str, set[str]] = {} for nid in node_set: - attrs = G.nodes[nid] + if not isinstance(nid, str): + continue + attrs = node_attrs[nid] sf = attrs.get("source_file") if not sf: continue @@ -667,6 +790,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat for alias_key, candidates in _alias_candidates.items(): if len(candidates) == 1: norm_to_id.setdefault(alias_key, next(iter(candidates))) + edge_by_pair: dict[object, EdgeData] = {} + # Iterate edges in a deterministic order. The graph is undirected and stores # direction in _src/_tgt; when two edges collapse onto the same node pair the # last write wins, so an unstable iteration order flips _src/_tgt run-to-run @@ -715,11 +840,11 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Sanitize numeric edge fields (#1960): an explicit ``"weight": null`` in # the extraction JSON survives ``.get("weight", 1.0)`` (the key is present, # so the default never applies) and reaches Louvain/Leiden as None, - # crashing modularity arithmetic with a TypeError (graspologic's Leiden + # crashing native modularity arithmetic with a TypeError (Leiden # even panics on NaN). Coerce to float and fall back to the schema default # of 1.0 for anything the clustering backends reject — None, non-numeric # strings, NaN/inf, negatives — while numeric strings coerce cleanly. - # Repair (not drop) the key so graph.json round-trips a clean value and a + # Repair (not drop) the key so native graph round-trips a clean value and a # cluster-only/--update reload never re-ingests the null. attrs = {k: v for k, v in edge.items() if k not in ("source", "target", "target_file")} for _num_key in ("weight", "confidence_score"): @@ -736,8 +861,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # flags and leaves query results with no file reference (#1279). if not attrs.get("source_file"): attrs["source_file"] = ( - G.nodes[src].get("source_file") - or G.nodes[tgt].get("source_file") + node_attrs[src].get("source_file") + or node_attrs[tgt].get("source_file") or "" ) if "source_file" in attrs: @@ -749,8 +874,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Python `import time` must not bind to a `time.ts`, #1749). _edge_rel = attrs.get("relation") if _edge_rel in ("calls", "imports", "imports_from", "references"): - src_ext = Path(G.nodes[src].get("source_file") or "").suffix.lower() - tgt_ext = Path(G.nodes[tgt].get("source_file") or "").suffix.lower() + src_ext = Path(node_attrs[src].get("source_file") or "").suffix.lower() + tgt_ext = Path(node_attrs[tgt].get("source_file") or "").suffix.lower() src_fam = _EDGE_LANG_FAMILY.get(src_ext) tgt_fam = _EDGE_LANG_FAMILY.get(tgt_ext) if _edge_rel == "calls": @@ -778,25 +903,29 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # earlier one's _src/_tgt, silently flipping the surviving edge's caller # and callee. First-seen direction wins instead — drop the redundant # reverse-direction duplicate so the original direction is preserved (#1061). - if not G.is_directed() and G.has_edge(src, tgt): - existing = edge_data(G, src, tgt) - if existing.get("relation") == attrs.get("relation") and ( - existing.get("_src") == tgt and existing.get("_tgt") == src + edge_key = edge.get("key") if multigraph else None + pair_base: object = (src, tgt) if directed else frozenset((src, tgt)) + pair: object = (pair_base, edge_key) if multigraph else pair_base + existing = edge_by_pair.get(pair) + if not directed and existing is not None: + existing_attrs = dict(existing.attributes) + if existing_attrs.get("relation") == attrs.get("relation") and ( + existing_attrs.get("_src") == tgt and existing_attrs.get("_tgt") == src ): continue - G.add_edge(src, tgt, **attrs) + edge_by_pair[pair] = EdgeData(src, tgt, attrs, edge_key) hyperedges = extraction.get("hyperedges", []) if hyperedges: # Relativize hyperedge source_file the same way nodes and edges are - # (above), so to_json — which has no root and writes G.graph["hyperedges"] - # verbatim — never leaks an absolute path from a semantic subagent (#1418). + # (above), so native persistence never leaks an absolute path from a + # semantic subagent (#1418). kept_hyperedges = [] for he in hyperedges: if isinstance(he, dict) and he.get("source_file"): he["source_file"] = _norm_source_file(he["source_file"], _root) # Validate members against the built node set (#1916): a hyperedge # member absent from the graph used to be copied into - # G.graph["hyperedges"] verbatim and reach graph.json dangling, + # G.graph["hyperedges"] verbatim and reach native graph dangling, # even from a live (non-cache) extraction. Mirror the pairwise-edge # handling above: remap mismatched ids via normalization first, # then drop members that still don't resolve; drop the hyperedge @@ -825,19 +954,30 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if valid_members != he["nodes"]: he["nodes"] = valid_members kept_hyperedges.append(he) - if kept_hyperedges: - G.graph["hyperedges"] = kept_hyperedges - return G + graph_attributes = {"hyperedges": kept_hyperedges} if hyperedges and kept_hyperedges else {} + kind = ( + "multidigraph" if directed and multigraph else + "digraph" if directed else + "multigraph" if multigraph else + "graph" + ) + return GraphBuildData( + kind=kind, + nodes=[NodeData(node_id, attrs) for node_id, attrs in node_attrs.items()], + edges=list(edge_by_pair.values()), + attributes=graph_attributes, + ) def build( extractions: list[dict], *, directed: bool = False, + multigraph: bool = False, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, -) -> nx.Graph: +) -> GraphBuildData: """Merge multiple extraction results into one graph. directed=True produces a DiGraph that preserves edge direction (source→target). @@ -848,7 +988,7 @@ def build( root: if given, absolute source_file paths are made relative to root (#932). Extractions are merged in order. For nodes with the same ID, the last - extraction's attributes win (NetworkX add_node overwrites). Pass AST + extraction's attributes win. Pass AST results before semantic results so semantic labels take precedence, or reverse the order if you prefer AST source_location precision to win. """ @@ -865,7 +1005,9 @@ def build( combined["nodes"], combined["edges"], communities={}, dedup_llm_backend=dedup_llm_backend, ) - return build_from_json(combined, directed=directed, root=root) + return build_from_extraction( + combined, directed=directed, multigraph=multigraph, root=root + ) def _norm_label(label: str | None) -> str: @@ -935,48 +1077,75 @@ def build_merge( graph_path: str | Path | None = None, prune_sources: list[str] | None = None, *, - directed: bool = False, + directed: bool | None = None, + multigraph: bool | None = None, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, -) -> nx.Graph: - """Load existing graph.json, merge new chunks into it, and save back. - - Re-extracted files REPLACE their prior contribution: any source_file present - in new_chunks is dropped from the loaded graph before merging, so a changed - file's stale nodes/edges don't accumulate. Files absent from new_chunks are - preserved unchanged; deleted files are removed via prune_sources. - Safe to call repeatedly. + base_graph: GraphBuildData | None = None, + base_root: str | Path | None = None, + replace_origin_sources: dict[str, set[str]] | None = None, +) -> GraphBuildData: + """Merge extraction chunks into an explicit transient extraction DTO. + + Production updates rebuild this DTO from the durable extraction cache. This + helper accepts an explicit transient base only; it never projects an active + ``NativeGraph`` back into Python. root: if given, absolute source_file paths in new_chunks are made relative (#932). """ - graph_path = Path(graph_path if graph_path is not None else _default_graph_json()) - if graph_path.exists(): - # Read JSON directly instead of going through node_link_graph(). - # The latter rebuilds an undirected nx.Graph and then enumerating - # edges() yields endpoints based on node insertion order, which - # silently flips directional edges (e.g. `calls`) when the callee - # was inserted before the caller. The _src/_tgt direction-preserving - # attrs are popped before saving in export.py, so going through the - # NetworkX round-trip loses direction permanently (#760). - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(graph_path) - try: - data = json.loads(graph_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - raise RuntimeError( - f"Cannot read {graph_path} for incremental merge: {exc}. " - "Delete the file and run a full rebuild." - ) from exc - links_key = "links" if "links" in data else "edges" - existing_nodes = list(data.get("nodes", [])) - existing_edges = list(data.get(links_key, [])) - existing_hyperedges = list(data.get("hyperedges", [])) + graph_path = Path(graph_path if graph_path is not None else DEFAULT_PROJECT_STORE) + if base_graph is not None: + existing_nodes = [ + {"id": node.id, **dict(node.attributes)} + for node in base_graph.nodes + ] + existing_edges = [ + { + "source": edge.source, + "target": edge.target, + **dict(edge.attributes), + **({"key": edge.key} if base_graph.multigraph else {}), + } + for edge in base_graph.edges + ] + raw_hyperedges = base_graph.attributes.get("hyperedges", []) + existing_hyperedges = list(raw_hyperedges) if isinstance(raw_hyperedges, list) else [] + if directed is None: + directed = base_graph.directed + if multigraph is None: + multigraph = base_graph.multigraph had_graph = True else: existing_nodes = [] existing_edges = [] existing_hyperedges = [] had_graph = False + if directed is None: + directed = False + if multigraph is None: + multigraph = False + + # Preserve source identity when invocation style changes (for example an + # absolute ``src`` build followed by a project-relative ``src`` update). + # The active generation's paths are relative to ``base_root``; fresh paths + # are relative to ``root``. Rebase only paths that fit under the new root. + old_root = Path(base_root).resolve() if base_root is not None else None + new_root = Path(root).resolve() if root is not None else None + if old_root is not None and new_root is not None and old_root != new_root: + def _rebase_source(item: dict) -> None: + source = item.get("source_file") + if not source: + return + source_path = Path(str(source)) + absolute = source_path if source_path.is_absolute() else old_root / source_path + try: + item["source_file"] = absolute.resolve().relative_to(new_root).as_posix() + except (OSError, ValueError): + return + + for item in [*existing_nodes, *existing_edges, *existing_hyperedges]: + if isinstance(item, dict): + _rebase_source(item) # Effective root for relativizing absolute source_file / prune paths back to the # stored relative source_file keys. When the caller passes root we use it; @@ -1001,26 +1170,63 @@ def build_merge( # absolute win32 paths while the stored graph keeps relative posix (#1007). _replace_root = _eff_root new_sources: set[str] = set() + replaced_origins: dict[str, set[str]] = {} + for source, origins in (replace_origin_sources or {}).items(): + identities = {source} + norm = _norm_source_file(source, _replace_root) + if norm: + identities.add(norm) + new_sources.update(identities) + for identity in identities: + replaced_origins.setdefault(identity, set()).update(origins) for ch in new_chunks: - for n in ch.get("nodes", []): - sf = n.get("source_file") - if not sf: - continue - new_sources.add(sf) - norm = _norm_source_file(sf, _replace_root) - if norm: - new_sources.add(norm) + for bucket in ("nodes", "edges", "hyperedges"): + for item in ch.get(bucket, []): + sf = item.get("source_file") + if not sf: + continue + identities = {sf} + norm = _norm_source_file(sf, _replace_root) + if norm: + identities.add(norm) + new_sources.update(identities) + origin = item.get("_origin") + if isinstance(origin, str) and origin: + for identity in identities: + replaced_origins.setdefault(identity, set()).add(origin) if new_sources: def _kept(item: dict) -> bool: sf = item.get("source_file") - return sf not in new_sources and _norm_source_file(sf, _replace_root) not in new_sources + norm = _norm_source_file(sf, _replace_root) + if sf not in new_sources and norm not in new_sources: + return True + origin = item.get("_origin") + if not isinstance(origin, str) or not origin: + return True + return ( + origin not in replaced_origins.get(sf, set()) + and origin not in replaced_origins.get(norm, set()) + ) existing_nodes = [n for n in existing_nodes if _kept(n)] existing_edges = [e for e in existing_edges if _kept(e)] + if prune_sources: + existing_nodes = [ + node + for node in existing_nodes + if node.get("source_file") or node.get("_origin") != "ast" + ] base = [{"nodes": existing_nodes, "edges": existing_edges}] if had_graph else [] all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) + G = build( + all_chunks, + directed=directed, + multigraph=multigraph, + dedup=dedup, + dedup_llm_backend=dedup_llm_backend, + root=root, + ) # Prune set for deleted source files — both the raw form (matches nodes that # kept absolute source_file) and the normalised relative form (matches nodes @@ -1058,22 +1264,36 @@ def _kept(item: dict) -> bool: continue sf = he.get("source_file") norm = _norm_source_file(sf, _eff_root) - if sf in new_sources or norm in new_sources: - continue # re-extracted — replaced by the new chunk's version + origin = he.get("_origin") + if ( + (sf in new_sources or norm in new_sources) + and isinstance(origin, str) + and ( + origin in replaced_origins.get(sf, set()) + or origin in replaced_origins.get(norm, set()) + ) + ): + continue # this extraction tier was replaced if sf in prune_set or norm in prune_set: continue # deleted — pruned carried.append(he) if carried: - from graphify.export import attach_hyperedges - attach_hyperedges(G, carried) + current = G.attributes.setdefault("hyperedges", []) + current_ids = { + item.get("id") for item in current if isinstance(item, dict) + } + current.extend( + item for item in carried + if item.get("id") not in current_ids + ) # Prune nodes and edges from deleted source files if prune_sources: - to_remove = [ - n for n, d in G.nodes(data=True) - if d.get("source_file") in prune_set - ] - G.remove_nodes_from(to_remove) + to_remove = { + node.id for node in G.nodes + if node.attributes.get("source_file") in prune_set + } + G.nodes = [node for node in G.nodes if node.id not in to_remove] n_files = len(prune_sources) n_nodes = len(to_remove) if n_nodes: @@ -1083,11 +1303,14 @@ def _kept(item: dict) -> bool: ) edges_to_remove = [ - (u, v) for u, v, d in G.edges(data=True) - if d.get("source_file") in prune_set + edge for edge in G.edges + if edge.source in to_remove + or edge.target in to_remove + or edge.attributes.get("source_file") in prune_set ] if edges_to_remove: - G.remove_edges_from(edges_to_remove) + removed = set(id(edge) for edge in edges_to_remove) + G.edges = [edge for edge in G.edges if id(edge) not in removed] print( f"[graphify] Pruned {len(edges_to_remove)} edge(s) from deleted source file(s).", file=sys.stderr, @@ -1104,7 +1327,7 @@ def _kept(item: dict) -> bool: # Skip when dedup or prune_sources is active — shrinkage is intentional there. if graph_path.exists() and not dedup and not prune_sources: existing_n = len(existing_nodes) - new_n = G.number_of_nodes() + new_n = G.node_count if new_n < existing_n: raise ValueError( f"graphify: build_merge would shrink graph from {existing_n} → {new_n} nodes. " @@ -1114,7 +1337,7 @@ def _kept(item: dict) -> bool: return G -def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: +def prefix_graph_for_global(G: GraphBuildData, repo_tag: str) -> GraphBuildData: """Return a copy of G with all node IDs prefixed with repo_tag::. Labels are preserved unchanged (for display). A 'local_id' attribute @@ -1122,22 +1345,39 @@ def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: rewritten to match the new prefixed IDs. The 'repo' attribute is set on every node. """ - relabel = {n: f"{repo_tag}::{n}" for n in G.nodes} - H = nx.relabel_nodes(G, relabel, copy=True) - for node, data in H.nodes(data=True): - data["repo"] = repo_tag - data.setdefault("local_id", node.split("::", 1)[1]) - return H + relabel = {node.id: f"{repo_tag}::{node.id}" for node in G.nodes} + nodes = [] + for node in G.nodes: + attributes = dict(node.attributes) + attributes["repo"] = repo_tag + attributes.setdefault("local_id", node.id) + nodes.append(NodeData(relabel[node.id], attributes)) + edges = [ + EdgeData( + relabel[edge.source], + relabel[edge.target], + dict(edge.attributes), + edge.key, + ) + for edge in G.edges + ] + return GraphBuildData( + kind=G.kind, + nodes=nodes, + edges=edges, + attributes=dict(G.attributes), + extras=dict(G.extras), + ) def distinct_repo_tags(graph_paths: "list[Path]") -> "list[str]": - """Return a unique, human-meaningful repo tag per input graph for merge-graphs. + """Return a unique, human-meaningful repo tag per input graph. The naive tag (the ``graphify-out`` parent dir name) is NOT unique across inputs: ``src/graphify-out`` and ``frontend/src/graphify-out`` both yield ``src``. Prefixing both node sets with ``src::`` then makes same-stem nodes (a backend ``src/app.js`` and a frontend ``App.jsx``, both bare ``app``) - collide, so ``nx.compose`` silently merges two unrelated entities and invents + collide, so a merge silently combines two unrelated entities and invents cross-runtime edges (#1729). Colliding tags are widened with their own parent dir (``frontend_src``), then an index suffix guarantees uniqueness so no two graphs ever share a prefix. @@ -1158,8 +1398,12 @@ def distinct_repo_tags(graph_paths: "list[Path]") -> "list[str]": return unique -def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int: - """Remove all nodes tagged with repo_tag from G in-place. Returns count removed.""" - to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag] - G.remove_nodes_from(to_remove) +def prune_repo_from_graph(G: GraphBuildData, repo_tag: str) -> int: + """Remove all records tagged with repo_tag from build data. Returns node count.""" + to_remove = {node.id for node in G.nodes if node.attributes.get("repo") == repo_tag} + G.nodes = [node for node in G.nodes if node.id not in to_remove] + G.edges = [ + edge for edge in G.edges + if edge.source not in to_remove and edge.target not in to_remove + ] return len(to_remove) diff --git a/graphify/cache.py b/graphify/cache.py index 0a86a7e80..59775b38a 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -1,808 +1,216 @@ -# per-file extraction cache - skip unchanged files on re-run +"""Extraction caches stored inside Helix generation state. + +The caller owns the cache dictionary and persists it under +``state["incremental"]["extraction_cache"]`` with the topology generation. +No cache sidecars are read or written. +""" + from __future__ import annotations -import atexit +import copy import hashlib -import json -import os import re -import tempfile -import warnings from collections.abc import Iterable from pathlib import Path +from typing import Any, Callable -# Output directory name — override with GRAPHIFY_OUT env var for worktrees or -# shared-output setups. Accepts a relative name ("graphify-out-feature") or an -# absolute path ("/shared/graphify-out"). Single source of truth in graphify.paths -# (#1423); re-exported here as _GRAPHIFY_OUT for the existing call sites. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT - -# AST cache entries are the output of graphify's own extractor code, so they -# are only valid for the version that wrote them: keying purely on file -# content means extractor fixes shipped in a new release keep serving stale -# pre-fix results. The AST cache is therefore namespaced by package version -# (cache/ast/v{version}/), with entries from other versions removed on first -# use. The semantic cache is deliberately NOT versioned — its entries are -# produced by the LLM from file contents, and invalidating them on every -# release would re-bill extraction for unchanged files. -try: - from importlib.metadata import version as _pkg_version - - _EXTRACTOR_VERSION = _pkg_version("graphifyy") -except Exception: - _EXTRACTOR_VERSION = "unknown" - -# Version dirs already swept this process — cleanup runs once per (base, version). -_cleaned_ast_dirs: set[str] = set() - - -def _cleanup_stale_ast_entries(ast_base: Path, current_dir: Path) -> None: - """Remove AST cache entries left behind by other graphify versions. - - Sweeps sibling ``v*/`` directories and unversioned ``*.json`` entries - (the pre-versioning layout) under ``cache/ast/``. Best-effort: failures - are ignored, stragglers are retried on the next run. - """ - key = str(current_dir) - if key in _cleaned_ast_dirs: - return - _cleaned_ast_dirs.add(key) - if not ast_base.is_dir(): - return - import shutil - - for child in ast_base.iterdir(): - if child == current_dir: - continue - try: - if child.is_dir() and child.name.startswith("v"): - shutil.rmtree(child, ignore_errors=True) - elif child.suffix == ".json": - child.unlink() - except OSError: - pass - - -# Semantic cache entries are LLM output, so they depend on the extraction prompt -# that produced them, not just on file contents. Keying purely on content means a -# release that changes the prompt keeps replaying entries from the older prompt on -# every unchanged file, silently mixing extraction vintages in one graph (#1939). -# Versioning them by package version (as the AST cache does) would re-bill LLM -# extraction on every patch release — the reason #1252 deliberately left them -# unversioned. Fingerprinting the prompt itself keeps both properties: entries -# survive releases that don't touch the prompt, and invalidate only when it -# actually changed. Entries live under cache/semantic/p{fingerprint}/ when the -# caller supplies its prompt; callers that don't keep the historical flat layout. -_PROMPT_FP_LEN = 12 - -# Count of pre-fingerprint (flat-layout) entries served this process, so -# check_semantic_cache can report N to the user (#1939). -_legacy_semantic_hits = 0 - -# Prompt-file fingerprints already computed, keyed by (path, size, mtime_ns) — -# the same stat signature the hash index uses. check_semantic_cache resolves the -# prompt once per FILE in the corpus, so without this a 500-doc run re-reads and -# re-hashes the same spec 500 times (and warns 500 times when it is unreadable). -_prompt_fp_cache: dict[tuple, str] = {} - - -def prompt_fingerprint(prompt: "str | Path") -> str: - """Return a short stable fingerprint of an extraction prompt. - ``prompt`` is either the prompt text itself (the Python extraction path owns - its system prompt, :func:`graphify.llm._extraction_system`) or a Path to the - prompt file an agent loaded (the skill path's - ``references/extraction-spec.md``). - - Line endings and trailing whitespace are normalized before hashing: the same - spec file checked out with CRLF on Windows must not fingerprint differently - from the LF checkout that wrote the cache, or every Windows run would look - like a prompt change and re-bill extraction. - """ - if isinstance(prompt, Path): - text = prompt.read_text(encoding="utf-8", errors="replace") - else: - text = prompt - normalized = "\n".join( - line.rstrip() for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") - ).strip() - return hashlib.sha256(normalized.encode()).hexdigest()[:_PROMPT_FP_LEN] - - -def _resolve_prompt_fp(prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None) -> str | None: - """Fingerprint the caller's extraction prompt, or None when it supplied none. - - ``prompt`` is prompt TEXT; ``prompt_file`` is a path to a file CONTAINING the - prompt. They are separate parameters rather than one overloaded argument - because the skill-driven callers are markdown snippets an agent copies with a - path substituted in — passing that path as ``prompt`` would hash the path - string itself, yielding a fingerprint that is stable, plausible, and tracks - nothing about the prompt. A silent wrong fingerprint is the exact failure - class #1939 is about, so the two are not inferred from each other. - - Best-effort: an unreadable ``prompt_file`` falls back to the flat, unattributed - layout rather than failing the run — a cache is never worth aborting an - extraction over. It warns rather than falling back quietly, because that - fallback silently restores the very behavior this fixes, and the skill-side - caller substitutes this path by hand. - """ - memo_key = None - if prompt_file is not None: - prompt = Path(prompt_file) - try: - st = prompt.stat() - memo_key = (str(prompt), st.st_size, st.st_mtime_ns) - if memo_key in _prompt_fp_cache: - return _prompt_fp_cache[memo_key] - except OSError: - pass # unreadable — fall through to the warning below - if prompt is None: - return None - try: - fp = prompt_fingerprint(prompt) - if memo_key is not None: - _prompt_fp_cache[memo_key] = fp - return fp - except (OSError, UnicodeError) as exc: - warnings.warn( - f"could not read extraction prompt {str(prompt)!r} ({exc}); semantic cache " - "entries cannot be attributed to a prompt version and fall back to the " - "unversioned layout, so this run may replay entries from an older " - "extraction prompt (#1939).", - RuntimeWarning, - stacklevel=3, - ) - return None - - -# A frontmatter delimiter is a whole line of exactly three dashes (optional -# trailing whitespace). Substring checks like startswith("---") / -# find("\n---") also match `----` thematic breaks and `--- text` prose, -# silently dropping everything above them from the hash (#1259). _FRONTMATTER_DELIM = re.compile(r"^---[ \t]*\r?$", re.MULTILINE) def _body_content(content: bytes) -> bytes: - """Strip YAML frontmatter from Markdown content, returning only the body.""" + """Strip a complete Markdown YAML frontmatter block for stable hashing.""" text = content.decode(errors="replace") opener = _FRONTMATTER_DELIM.match(text) if opener is None: return content closer = _FRONTMATTER_DELIM.search(text, opener.end()) - if closer is None: - return content - # Slice right after the closing `---` (not after its line) so the output - # stays byte-identical with the historical implementation for well-formed - # frontmatter -- existing semantic-cache hashes must not churn. - return text[closer.start() + 3:].encode() - + return text[closer.start() + 3:].encode() if closer is not None else content -# Stat-based index: maps absolute path → {size, mtime_ns, hash}. -# Loaded once per process, flushed via atexit. Skips full file reads when -# size+mtime_ns are unchanged — same trade-off as make(1). -# Correctness risks: `touch` causes a harmless extra re-hash; same-size edits -# within NFS second-resolution mtime have a 1-second window (same as make). -# `graphify extract --force` / `graphify update --force` (or GRAPHIFY_FORCE=1) -# skip the cache reads and re-dispatch everything when needed (#1894). -_stat_index: dict[str, dict] = {} -_stat_index_root: Path | None = None -_stat_index_dirty: bool = False +def file_hash(path: Path, root: Path = Path(".")) -> str: + """Hash file content; Markdown metadata-only changes do not invalidate extraction.""" + content = Path(path).read_bytes() + if Path(path).suffix.lower() in {".md", ".mdx"}: + content = _body_content(content) + return hashlib.sha256(content).hexdigest() -def _stat_index_file(root: Path) -> Path: - _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out - return base / "cache" / "stat-index.json" +def prompt_fingerprint(prompt: str | Path) -> str: + text = Path(prompt).read_text(encoding="utf-8", errors="replace") if isinstance(prompt, Path) else prompt + normalized = "\n".join( + line.rstrip() + for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + ).strip() + return hashlib.sha256(normalized.encode()).hexdigest()[:12] -def _ensure_stat_index(root: Path, cache_root: "Path | None" = None) -> None: - global _stat_index, _stat_index_root, _stat_index_dirty - if _stat_index_root is not None: - return - # The stat index only determines the cache FILE location (entry keys are - # absolute paths), so honoring an explicit cache_root keeps detect()'s - # word-count cache under the requested --out dir instead of polluting the - # scanned corpus with a stray graphify-out/ (#1747). - _stat_index_root = Path(cache_root if cache_root is not None else root).resolve() - p = _stat_index_file(_stat_index_root) - if p.exists(): - try: - _stat_index = json.loads(p.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - _stat_index = {} - else: - _stat_index = {} - atexit.register(_flush_stat_index) - - -def _flush_stat_index() -> None: - global _stat_index_dirty, _stat_index_root - if not _stat_index_dirty or _stat_index_root is None: - return - p = _stat_index_file(_stat_index_root) - try: - p.parent.mkdir(parents=True, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=p.parent, prefix="stat-index.", suffix=".tmp") - try: - os.write(fd, json.dumps(_stat_index, separators=(",", ":")).encode()) - os.close(fd) - os.replace(tmp, p) - except Exception: - try: - os.close(fd) - except OSError: - pass - try: - os.unlink(tmp) - except OSError: - pass - except OSError: - pass - _stat_index_dirty = False - - -def _normalize_path(path: Path) -> Path: - """Normalize path for consistent cache keys across Windows path spellings.""" - import sys - if sys.platform != "win32": - return path - s = str(path) - if s.startswith("\\\\?\\"): - s = s[4:] # strip extended-length prefix \\?\ - return Path(os.path.normcase(s)) - - -def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = None) -> str: - """SHA256 of file contents + path relative to root. - - Uses a stat-based fastpath (size + mtime_ns) to skip full reads when the - file hasn't changed. Falls through to full SHA256 on first encounter or - when stat changes. Index is flushed atomically at process exit. - Using a relative path (not absolute) makes cache entries portable across - machines and checkout directories, so shared caches and CI work correctly. - Falls back to the resolved absolute path if the file is outside root. +def _prompt_fp(prompt: str | Path | None, prompt_file: str | Path | None) -> str: + if prompt_file is not None: + return prompt_fingerprint(Path(prompt_file)) + return prompt_fingerprint(prompt) if prompt is not None else "unscoped" - For Markdown files (.md), only the body below the YAML frontmatter is hashed, - so metadata-only changes (e.g. reviewed, status, tags) do not invalidate the cache. - """ - global _stat_index_dirty - p = _normalize_path(Path(path)) - root = _normalize_path(Path(root)) - if not p.is_file(): - raise IsADirectoryError(f"file_hash requires a file, got: {p}") - # The stat index is a cache artifact, so it must follow the cache location - # (cache_root), not the key-anchor root — otherwise it leaves a stray - # graphify-out/cache/stat-index.json inside the analyzed source tree even when - # the AST cache itself is redirected to CWD (#1774 completion). - _ensure_stat_index(root, cache_root=cache_root) - resolved = p.resolve() - abs_key = str(resolved) - # The salt is the path component that enters the digest (relative to root, or - # the absolute-path fallback). The stat-index memo MUST be keyed by it too: - # the same file hashed under two different roots yields two different digests - # (this happens within one `--out` run), and a memo keyed only by absolute - # path served whichever was computed first — making file_hash order-dependent - # and poisoning the persisted stat-index across runs (#1989). Store one digest - # per salt so alternating roots don't force re-reads. +def _relative(path: Path, root: Path) -> str: + resolved = path.resolve() try: - salt = resolved.relative_to(Path(root).resolve()).as_posix().lower() + return resolved.relative_to(root.resolve()).as_posix() except ValueError: - salt = resolved.as_posix().lower() + return resolved.as_posix() - st: "os.stat_result | None" = None - try: - st = p.stat() - entry = _stat_index.get(abs_key) - if (isinstance(entry, dict) - and entry.get("size") == st.st_size - and entry.get("mtime_ns") == st.st_mtime_ns): - hashes = entry.get("hashes") - if isinstance(hashes, dict): - cached = hashes.get(salt) - if isinstance(cached, str): - return cached - # Legacy single-digest entries ("hash") don't record which salt - # produced them, so they are never trusted (#1989) — recompute once. - except OSError: - pass - raw = p.read_bytes() - content = _body_content(raw) if p.suffix.lower() == ".md" else raw - h = hashlib.sha256() - h.update(content) - h.update(b"\x00") - h.update(salt.encode()) - digest = h.hexdigest() - - if st is not None: - entry = _stat_index.get(abs_key) - if (isinstance(entry, dict) - and entry.get("size") == st.st_size - and entry.get("mtime_ns") == st.st_mtime_ns): - hashes = entry.get("hashes") - if not isinstance(hashes, dict): - hashes = {} - entry["hashes"] = hashes - hashes[salt] = digest # preserve a co-located word_count / other salts - entry.pop("hash", None) # retire the un-salted legacy digest - else: - _stat_index[abs_key] = {"size": st.st_size, "mtime_ns": st.st_mtime_ns, - "hashes": {salt: digest}} - _stat_index_dirty = True +def _cache_key( + path: Path, + root: Path, + kind: str, + prompt: str | Path | None, + prompt_file: str | Path | None, +) -> str: + return f"{kind}:{_prompt_fp(prompt, prompt_file)}:{_relative(path, root)}" - return digest - - -def cached_word_count(path: Path, root: Path, compute, cache_root: "Path | None" = None) -> int: - """Word count with the same (size, mtime_ns) stat-fastpath cache as - :func:`file_hash`, persisted in the shared stat index. - - ``detect()`` counts words in every PDF/docx/text file to size the corpus, - which re-opens and re-parses every binary on each run — minutes on a large - docs corpus even when only a handful of files changed (#1656). This caches - the count against the file's stat signature so an unchanged file is counted - once and read from the index thereafter. ``compute(path)`` produces the - count on a miss. A file that can't be stat'd (e.g. a Windows long path the - index normalization can't reach) simply recomputes and isn't cached — - correct, just not accelerated. - """ - global _stat_index_dirty - p = _normalize_path(Path(path)) - root = _normalize_path(Path(root)) - _ensure_stat_index(root, cache_root=cache_root) - abs_key = str(p.resolve()) - st: "os.stat_result | None" = None - try: - st = p.stat() - entry = _stat_index.get(abs_key) - if (entry - and entry.get("size") == st.st_size - and entry.get("mtime_ns") == st.st_mtime_ns - and "word_count" in entry): - return entry["word_count"] - except OSError: - pass - wc = compute(Path(path)) - - if st is not None: - entry = _stat_index.get(abs_key) - if (entry - and entry.get("size") == st.st_size - and entry.get("mtime_ns") == st.st_mtime_ns): - entry["word_count"] = wc # augment the existing hash entry in place - else: - _stat_index[abs_key] = { - "size": st.st_size, "mtime_ns": st.st_mtime_ns, "word_count": wc, - } - _stat_index_dirty = True - - return wc - - -def _relativize_source_files_in(payload: dict, root: Path) -> None: - """Mutate ``payload`` to rewrite absolute ``source_file`` fields as - forward-slash relative paths from ``root``. - - Mirror of :func:`graphify.watch._relativize_source_files` so cached - extraction fragments persist in portable form (#777). Already-relative - fields and out-of-root paths pass through unchanged. - - Only ``root`` is resolved — ``source_file`` itself is relativized - symbolically so in-root symlinks keep their original name rather than - pointing at the resolved target. Same reasoning as - :func:`graphify.detect._to_relative_for_storage`. - """ - try: - root_resolved = Path(root).resolve() - except OSError: - return - # raw_calls (#: Pascal/Delphi cross-file inherited-call resolution) carries - # source_file the same way nodes/edges/hyperedges do, so it needs the same - # portable-path treatment for cache entries to round-trip correctly across - # machines/checkout directories. - for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): - for item in payload.get(bucket, []): - if not isinstance(item, dict): - continue - source = item.get("source_file") - if not source: - continue - sp = Path(source) - if not sp.is_absolute(): +def _portable_result(result: dict[str, Any], root: Path) -> dict[str, Any]: + value = copy.deepcopy(result) + for bucket in ("nodes", "edges", "hyperedges"): + for item in value.get(bucket, []): + if not isinstance(item, dict) or not item.get("source_file"): continue - try: - rel = os.path.relpath(sp, root_resolved) - except (ValueError, OSError): - continue # out-of-root (e.g. Windows cross-drive) - if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): - continue # escaped root — keep absolute - item["source_file"] = rel.replace(os.sep, "/") - + source = Path(str(item["source_file"])) + if source.is_absolute(): + item["source_file"] = _relative(source, root) + return value -def _absolutize_source_files_in(payload: dict, root: Path) -> None: - """Inverse of :func:`_relativize_source_files_in`. - Re-anchor relative ``source_file`` fields against ``root`` so callers - that load a cached fragment see the same absolute-path shape that a - fresh in-process extraction would produce. Legacy cache entries with - absolute ``source_file`` values pass through unchanged. - """ - try: - root_resolved = Path(root).resolve() - except OSError: - return - for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): - for item in payload.get(bucket, []): - if not isinstance(item, dict): - continue - source = item.get("source_file") - if not source: - continue - sp = Path(source) - if sp.is_absolute(): - continue - try: - item["source_file"] = str(root_resolved / sp) - except (TypeError, OSError): +def _runtime_result(result: dict[str, Any], root: Path) -> dict[str, Any]: + """Restore the absolute source paths emitted by fresh AST extraction.""" + value = copy.deepcopy(result) + for bucket in ("nodes", "edges", "hyperedges"): + for item in value.get(bucket, []): + if not isinstance(item, dict) or not item.get("source_file"): continue + source = Path(str(item["source_file"])) + if not source.is_absolute(): + item["source_file"] = str((root / source).resolve()) + return value -def cache_dir(root: Path = Path("."), kind: str = "ast", - prompt_fp: str | None = None) -> Path: - """Returns the cache directory for ``kind`` - creates it if needed. - - kind is "ast", "semantic", or a mode-namespaced semantic kind such as - "semantic-deep" (#1894). Separate subdirectories prevent semantic cache - entries from overwriting AST cache entries for the same source_file (#582). - - AST entries live in graphify-out/cache/ast/v{version}/ — namespaced by - graphify version because they depend on extractor code, not just file - contents. Semantic entries are still NOT version-namespaced (re-extraction - costs LLM calls, #1252): they live in graphify-out/cache/semantic/, with - deep-mode entries beside them in graphify-out/cache/semantic-deep/. - - ``prompt_fp`` (semantic kinds only) adds a p{fingerprint}/ subdirectory so - entries are attributed to the extraction prompt that produced them (#1939). - Omitting it yields the historical flat layout, where entries of unknown - vintage live. - """ - _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out - d = base / "cache" / kind - if kind == "ast": - d = d / f"v{_EXTRACTOR_VERSION}" - _cleanup_stale_ast_entries(d.parent, d) - elif prompt_fp: - d = d / f"p{prompt_fp}" - d.mkdir(parents=True, exist_ok=True) - return d - - -def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", - cache_root: Path | None = None, prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None, - allow_legacy: bool = True, - allow_partial: bool = False) -> dict | None: - """Return cached extraction for this file if hash matches, else None. - - Cache key: SHA256 of file contents. - Cache value: stored as graphify-out/cache/{kind}/{hash}.json (AST entries - under the per-version subdirectory, see :func:`cache_dir`). - - ``root`` anchors the content-hash key and source_file relativization (it - must stay the inferred common parent so keys remain portable). ``cache_root`` - decouples *where* the cache directory lives from that anchor — the cache is - an output and must not land inside a read-only/analyzed source tree (#1774). - When ``cache_root`` is None the location falls back to ``root`` (unchanged - behavior for existing callers). +def _group_has_partial_marker(result: dict[str, Any]) -> bool: + return any( + isinstance(item, dict) and item.get("_partial") is True + for bucket in ("nodes", "edges", "hyperedges") + for item in result.get(bucket, []) + ) - AST entries written by other graphify versions — including the legacy - flat cache/ layout (pre-0.5.3) and the unversioned cache/ast/ layout — - are deliberately not consulted: they were produced by a different - extractor and may be stale. - ``prompt`` (semantic kinds) is the extraction prompt — text, or a Path to - the prompt file — that the caller is about to extract with. It selects the - p{fingerprint}/ namespace, so an entry produced by a different prompt is a - miss rather than a silent stale hit (#1939). When it is given and the - fingerprinted namespace misses, ``allow_legacy`` (default True) falls back - to a flat-layout entry: those predate fingerprinting, so their vintage is - unknowable — they are served rather than re-billed, and the hit is counted - so :func:`check_semantic_cache` can report N to the user. Callers that must - not mix vintages within one entry (see :func:`save_semantic_cache`'s - ``merge_existing``) pass allow_legacy=False. - Returns None if no cache entry or file has changed. - """ - global _legacy_semantic_hits - location = cache_root if cache_root is not None else root - try: - h = file_hash(path, root, cache_root=cache_root) - except OSError: +def load_cached( + path: Path, + root: Path = Path("."), + kind: str = "ast", + cache_root: Path | None = None, + prompt: str | Path | None = None, + prompt_file: str | Path | None = None, + allow_legacy: bool = True, + allow_partial: bool = False, + allow_stale: bool = False, + *, + cache: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any] | None: + del cache_root, allow_legacy + if cache is None or not Path(path).is_file(): return None - prompt_fp = _resolve_prompt_fp(prompt, prompt_file) - entry = cache_dir(location, kind, prompt_fp) / f"{h}.json" - legacy_hit = False - if prompt_fp and not entry.exists() and allow_legacy: - legacy = cache_dir(location, kind) / f"{h}.json" - if legacy.exists(): - entry, legacy_hit = legacy, True - if entry.exists(): - try: - result = json.loads(entry.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return None - # A ``partial`` entry was produced from a truncated LLM response and - # covers only part of the file's symbols. Serving it as authoritative - # would return the incomplete node set forever until the file is - # re-extracted. Treat it as a cache MISS (the normal read path) so the - # file is re-dispatched and retried. Self-heals: a later complete - # extraction overwrites the same content-hash key with a non-partial - # entry. ``allow_partial`` is the one exception — the merge_existing - # checkpoint peeks at a partial prev so it can accumulate a file's slices - # across chunks without losing the truncated one (it stays partial). - if not allow_partial and isinstance(result, dict) and result.get("partial"): - return None - if legacy_hit: - _legacy_semantic_hits += 1 - # Re-anchor relative source_file fields so callers see the same - # absolute-path shape that a fresh in-process extraction produces - # (#777). Legacy entries with absolute source_file pass through. - if isinstance(result, dict): - _absolutize_source_files_in(result, root) - return result - return None - - -def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "ast", - cache_root: Path | None = None, prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None) -> None: - """Save extraction result for this file. - - Stores as graphify-out/cache/{kind}/{hash}.json where hash = SHA256 of current file contents. - result should be a dict with 'nodes' and 'edges' lists. - - ``root`` anchors the content-hash key and source_file relativization; - ``cache_root`` (when given) is where the cache directory is written, decoupled - from ``root`` so the cache never lands inside the analyzed source tree (#1774). + key = _cache_key(Path(path), Path(root), kind, prompt, prompt_file) + entry = cache.get(key) + if not isinstance(entry, dict) or ( + not allow_stale + and entry.get("content_hash") != file_hash(Path(path), Path(root)) + ): + return None + if entry.get("partial") and not allow_partial: + return None + result = entry.get("result") + return _runtime_result(result, Path(root)) if isinstance(result, dict) else None - ``prompt`` (semantic kinds) is the extraction prompt that produced ``result`` - — text, or a Path to the prompt file. It stamps the entry into the - p{fingerprint}/ namespace so a later run under a different prompt does not - replay it (#1939). Writes always land in the fingerprinted namespace when a - prompt is given: an entry of known vintage is never written back into the - flat unknown-vintage layout. - No-ops if `path` is not a regular file. Subagent-produced semantic fragments - occasionally carry a directory path in `source_file`; skipping them prevents - IsADirectoryError from aborting the whole batch. - """ - p = Path(path) - if not p.is_file(): +def save_cached( + path: Path, + result: dict[str, Any], + root: Path = Path("."), + kind: str = "ast", + cache_root: Path | None = None, + prompt: str | Path | None = None, + prompt_file: str | Path | None = None, + *, + cache: dict[str, dict[str, Any]] | None = None, + partial: bool | None = None, +) -> None: + del cache_root + if cache is None or not Path(path).is_file(): return - # Relativize source_file fields against ``root`` before write so the - # cache file on disk is portable across machines and checkout - # directories (#777). The cache key is content-hashed so lookup is - # already path-independent; this fixes the embedded path leak. - # - # Serialize a relativized copy rather than mutating the caller's dict — - # downstream pipeline steps (notably extract.py's AST prefix remap, which - # looks up Path(source_file).resolve() in a prefix table) depend on the - # source_file field's original absolute form. Mutating the input here would - # silently break those remaps on the first extraction pass. - on_disk = result - if isinstance(result, dict) and any(result.get(k) for k in ("nodes", "edges", "hyperedges", "raw_calls")): - import copy as _copy - on_disk = _copy.deepcopy(result) - _relativize_source_files_in(on_disk, root) - h = file_hash(p, root, cache_root=cache_root) - location = cache_root if cache_root is not None else root - target_dir = cache_dir(location, kind, _resolve_prompt_fp(prompt, prompt_file)) - entry = target_dir / f"{h}.json" - fd, tmp_path = tempfile.mkstemp(dir=target_dir, prefix=f"{h}.", suffix=".tmp") - try: - os.write(fd, json.dumps(on_disk).encode()) - os.close(fd) - try: - os.replace(tmp_path, entry) - except PermissionError: - # Windows: os.replace can fail with WinError 5 if the target is - # briefly locked. Fall back to copy-then-delete. - import shutil - shutil.copy2(tmp_path, entry) - os.unlink(tmp_path) - except Exception: - try: - os.close(fd) - except OSError: - pass - try: - os.unlink(tmp_path) - except OSError: - pass - raise - - -def cached_files(root: Path = Path(".")) -> set[str]: - """Return set of file hashes that have a valid cache entry (any kind).""" - base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" - hashes: set[str] = set() - # Legacy flat entries - if base.is_dir(): - hashes.update(p.stem for p in base.glob("*.json")) - # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, - # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds - # have per-prompt-fingerprint subdirs alongside pre-fingerprint flat entries - # (#1939). - for kind in ("ast", "semantic", "semantic-deep"): - d = base / kind - if d.is_dir(): - hashes.update(p.stem for p in d.glob("**/*.json")) - return hashes - - -def clear_cache(root: Path = Path(".")) -> None: - """Delete all cache entries (ast/, semantic/, semantic-deep/, and legacy - flat entries).""" - base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" - # Legacy flat entries - if base.is_dir(): - for f in base.glob("*.json"): - f.unlink() - # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, - # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds - # have per-prompt-fingerprint subdirs (#1939). - for kind in ("ast", "semantic", "semantic-deep"): - d = base / kind - if d.is_dir(): - for f in d.glob("**/*.json"): - f.unlink() + key = _cache_key(Path(path), Path(root), kind, prompt, prompt_file) + portable = _portable_result(result, Path(root)) + cache[key] = { + "content_hash": file_hash(Path(path), Path(root)), + "kind": kind, + "prompt_fingerprint": _prompt_fp(prompt, prompt_file), + "partial": _group_has_partial_marker(portable) if partial is None else partial, + "result": portable, + } -def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: - """Remove orphaned semantic cache entries, returning the count pruned. +def cached_files(cache: dict[str, dict[str, Any]]) -> set[str]: + return {str(entry.get("content_hash")) for entry in cache.values() if entry.get("content_hash")} - The semantic cache is content-hash-keyed (``{file_hash}.json`` under - ``cache/semantic/``) and deliberately UNVERSIONED — entries are produced by - the LLM from file contents, so invalidating them on every release would - re-bill extraction. Because it is unversioned it is also never swept by the - AST version-cleanup, so every content change or file deletion leaves a - permanent orphan entry that accumulates unbounded. - This sweeps ``cache/semantic/*.json`` AND ``cache/semantic-deep/*.json`` - (the ``--mode deep`` namespace, #1894) and deletes any entry whose stem - (the content hash) is not in ``live_hashes`` — the hashes of the current - live document set. Both namespaces are pruned against the SAME live set: - liveness is content-based and mode-independent, so a hash that is live for - one namespace is live for both. Skipping the deep namespace would re-grow - the unbounded-orphan problem this function fixed (#1527). ``*.tmp`` - atomic-write temporaries are skipped, and only these directories are - touched (never ``cache/ast/**`` or anything else). The unversioned design - is preserved: we prune by liveness, not by version. +def clear_cache(cache: dict[str, dict[str, Any]]) -> None: + cache.clear() - The sweep recurses into the per-prompt-fingerprint subdirs (#1939) for the - same reason it covers the deep namespace: a glob that stopped at the top - level would leave every fingerprinted entry permanently unprunable. Entries - under a fingerprint other than the current one are pruned by liveness only, - never swept wholesale the way :func:`_cleanup_stale_ast_entries` sweeps old - AST versions — two hosts with different prompts (verbose vs compact - extraction-spec) can share one graphify-out/, and a wholesale sweep would - have each run delete the other's entries and re-bill extraction on every - alternation. Liveness keeps the total bounded by live docs × prompts seen. - Best-effort, mirroring :func:`_cleanup_stale_ast_entries`: each unlink is - wrapped in ``try/except OSError`` and a failure is ignored. The worst-case - failure mode is benign — a surviving orphan costs only one re-extraction of - one doc on a future run, never incorrect output. - """ - _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out - pruned = 0 - for kind in ("semantic", "semantic-deep"): - semantic_dir = base / "cache" / kind - if not semantic_dir.is_dir(): - continue - for entry in semantic_dir.glob("**/*.json"): - if entry.stem in live_hashes: - continue - try: - entry.unlink() - pruned += 1 - except OSError: - pass - return pruned +def cached_word_count( + path: Path, + root: Path, + compute: Callable[[Path], int], + cache_root: Path | None = None, +) -> int: + """Word counts are cheap and are not persisted outside the active generation.""" + del root, cache_root + return int(compute(path)) def check_semantic_cache( files: list[str], + cache: dict[str, dict[str, Any]], + *, root: Path = Path("."), mode: str | None = None, - prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None, - cache_root: "Path | None" = None, + prompt: str | Path | None = None, + prompt_file: str | Path | None = None, + allow_stale: bool = False, ) -> tuple[list[dict], list[dict], list[dict], list[str]]: - """Check semantic extraction cache for a list of absolute file paths. - - Returns (cached_nodes, cached_edges, cached_hyperedges, uncached_files). - Uncached files need Claude extraction; cached files are merged directly. - - ``mode`` selects the cache namespace: ``None`` (the default) reads - ``cache/semantic/`` — byte-identical to the historical behavior, so - existing callers that omit it (including older installed skill flows) - are unaffected. A non-None mode (e.g. ``"deep"``) reads - ``cache/semantic-{mode}/`` instead, so deep-mode results never shadow - (or get shadowed by) standard-mode entries for the same content (#1894). - - ``prompt`` is the extraction prompt this run will use for the uncached - files — the prompt text (Python path) or a Path to the prompt file the - agent loaded (skill path, ``references/extraction-spec.md``). Supplying it - restricts hits to entries produced by that same prompt, so an upgrade that - changed the prompt re-extracts instead of replaying the older vintage - (#1939). Entries written before fingerprinting existed still hit — their - vintage is unknowable and dropping them would re-bill a whole corpus — but - a warning reports how many were served. Omitting ``prompt`` keeps the - historical behavior for existing callers. - - ``cache_root`` decouples *where* the cache is read from the key-anchor - ``root``, mirroring :func:`load_cached` and :func:`save_semantic_cache` - (#1774 / #1990). With ``--out``, pass the corpus as ``root`` (so content-hash - keys and relative-path resolution stay anchored to the source tree) and the - output directory as ``cache_root``. Omitting it keeps ``root`` for both. - """ - global _legacy_semantic_hits kind = "semantic" if mode is None else f"semantic-{mode}" - cached_nodes: list[dict] = [] - cached_edges: list[dict] = [] - cached_hyperedges: list[dict] = [] + nodes: list[dict] = [] + edges: list[dict] = [] + hyperedges: list[dict] = [] uncached: list[str] = [] - legacy_before = _legacy_semantic_hits - - for fpath in files: - p = Path(fpath) - if not p.is_absolute(): - p = Path(root) / p - result = load_cached(p, root, kind=kind, cache_root=cache_root, - prompt=prompt, prompt_file=prompt_file) - if result is not None: - cached_nodes.extend(result.get("nodes", [])) - cached_edges.extend(result.get("edges", [])) - cached_hyperedges.extend(result.get("hyperedges", [])) - else: - uncached.append(fpath) - - legacy = _legacy_semantic_hits - legacy_before - if legacy: - warnings.warn( - f"{legacy} semantic cache entr{'y' if legacy == 1 else 'ies'} predate " - "extraction-prompt fingerprinting and were written by an unknown prompt " - "version; they were replayed as-is, so this graph may mix extraction " - "vintages. Re-run with --force (or GRAPHIFY_FORCE=1) to re-extract them " - "with the current prompt (#1939).", - RuntimeWarning, - stacklevel=2, + for raw in files: + path = Path(raw) + if not path.is_absolute(): + path = Path(root) / path + result = load_cached( + path, + root, + kind=kind, + prompt=prompt, + prompt_file=prompt_file, + allow_stale=allow_stale, + cache=cache, ) - - return cached_nodes, cached_edges, cached_hyperedges, uncached - - -def _group_has_partial_marker(group: dict) -> bool: - """True if any node/edge/hyperedge in a per-file group carries the internal - ``_partial`` truncation marker set by the adaptive-retry give-up sites. - - The marker rides the item dicts up through every chunk merge, so it reaches - ``save_semantic_cache`` on BOTH the incremental checkpoint path (llm.py) and - the final authoritative save (cli.py) without either caller having to thread - an extra argument — the final save would otherwise overwrite a checkpoint's - ``partial`` flag with a clean-looking entry. - """ - for bucket in ("nodes", "edges", "hyperedges"): - for item in group.get(bucket, []): - if isinstance(item, dict) and item.get("_partial"): - return True - return False + if result is None: + uncached.append(raw) + else: + nodes.extend(result.get("nodes", [])) + edges.extend(result.get("edges", [])) + hyperedges.extend(result.get("hyperedges", [])) + return nodes, edges, hyperedges, uncached def save_semantic_cache( @@ -810,231 +218,91 @@ def save_semantic_cache( edges: list[dict], hyperedges: list[dict] | None = None, root: Path = Path("."), + cache_root: Path | None = None, merge_existing: bool = False, allowed_source_files: Iterable[str | Path] | None = None, mode: str | None = None, - prompt: "str | Path | None" = None, - prompt_file: "str | Path | None" = None, + prompt: str | Path | None = None, + prompt_file: str | Path | None = None, partial_source_files: Iterable[str | Path] | None = None, - cache_root: "Path | None" = None, + *, + cache: dict[str, dict[str, Any]] | None = None, ) -> int: - """Save semantic extraction results to cache, keyed by source_file. - - Groups nodes and edges by source_file, then saves one cache entry per file - under cache/semantic/ (separate from AST entries in cache/ast/) to prevent - hash-key collisions (#582). - - ``mode`` selects the cache namespace, mirroring - :func:`check_semantic_cache`: ``None`` (the default) writes - ``cache/semantic/`` — byte-identical to the historical behavior for - existing callers that omit it — while a non-None mode (e.g. ``"deep"``) - writes ``cache/semantic-{mode}/`` so richer deep-mode results never - overwrite standard-mode entries and vice versa (#1894). - - When ``merge_existing`` is True, any already-cached entry for a file is - unioned with the new results before saving instead of being overwritten. - This lets callers checkpoint incrementally (e.g. once per chunk) without - dropping a prior slice of a large file that was split across chunks. - - When ``allowed_source_files`` is provided, only those files may be used as - cache-write keys. Semantic nodes can legitimately mention another corpus - file, but a model must not be able to replace that file's complete cache - entry unless the file was part of the current extraction batch (#1757). - - When ``partial_source_files`` is provided, entries for those files are - stamped ``partial: True`` — the extraction was truncated, so the entry is - incomplete and :func:`load_cached` must treat it as a miss. Partial-ness is - ALSO detected intrinsically from a ``_partial`` marker on any grouped item, - so the flag survives even when a caller (e.g. cli.py's final save) does not - pass ``partial_source_files``. - - ``prompt`` is the extraction prompt that produced these results — text, or - a Path to the prompt file. It stamps entries into the p{fingerprint}/ - namespace so a later run under a different prompt re-extracts rather than - replaying them (#1939). Pass the same prompt here as to - :func:`check_semantic_cache`, or the write lands in a namespace the next - read won't consult. - - ``cache_root`` decouples *where* the cache directory is written from the - source-key anchor ``root`` — mirroring the same split that :func:`load_cached` - and :func:`save_cached` already expose (#1774). When given, cache files land - under ``cache_root`` while ``source_file`` paths are still resolved and - relativized against ``root``. When omitted, ``root`` is used for both - purposes (unchanged behaviour for existing callers). This fixes checkpoints - and the final save going to the corpus tree instead of ``--out`` (#1990, - #1991). - - Returns the number of files cached. - """ - from collections import defaultdict - + del cache_root + if cache is None: + return 0 kind = "semantic" if mode is None else f"semantic-{mode}" - by_file: dict[str, dict] = defaultdict(lambda: {"nodes": [], "edges": [], "hyperedges": []}) - for n in nodes: - src = n.get("source_file", "") - if src: - by_file[src]["nodes"].append(n) - for e in edges: - src = e.get("source_file", "") - if src: - by_file[src]["edges"].append(e) - for h in (hyperedges or []): - src = h.get("source_file", "") - if src: - by_file[src]["hyperedges"].append(h) - - root_path = Path(root).resolve() - - def resolved_source_path(value: str | Path) -> Path: - path = Path(value) - if not path.is_absolute(): - path = root_path / path - try: - return path.resolve() - except (OSError, RuntimeError): - # Keep the cache write best-effort for inaccessible paths or a - # symlink loop emitted by an untrusted semantic result. - return Path(os.path.abspath(path)) - - allowed_paths = None - if allowed_source_files is not None: - allowed_paths = {resolved_source_path(path) for path in allowed_source_files} - - partial_paths = None - if partial_source_files is not None: - partial_paths = {resolved_source_path(path) for path in partial_source_files} - # A chunk that truncated to an EMPTY parse contributes no grouped items, - # so its file is absent from by_file and the write loop below would never - # stamp it partial — leaving a prior clean slice looking complete (#1950 - # empty-parse gap). Seed an empty group for each named partial file that - # isn't already present, so the loop merges its existing entry and stamps - # it partial. Keyed by the resolved path (deduped against present groups). - _present = {resolved_source_path(k) for k in by_file} - for _pp in partial_paths: - if _pp not in _present: - by_file[str(_pp)] # defaultdict: create an empty {nodes,edges,hyperedges} - - def group_skipped(fpath: str) -> bool: - """Mirror the write-loop skip condition for one source_file group.""" - p = resolved_source_path(fpath) - return not p.is_file() or (allowed_paths is not None and p not in allowed_paths) - - # Dangling-reference pruning (#1916). A node group is skipped by the write - # loop below when its source_file is not a real file (ghost path) or is - # out-of-scope per the #1757 guard — but an edge/hyperedge in an ALLOWED - # group that references a node id from a skipped group used to be written - # verbatim, so on replay (check_semantic_cache) it dangled forever (the - # #1895 merged-result filter runs AFTER this checkpoint write and is - # bypassed entirely on replay). Compute the node ids that will be skipped - # and drop any to-be-written edge whose endpoint — or hyperedge whose - # member (whole-hyperedge drop, mirroring #1895) — references one. Gated - # on allowed_source_files so unscoped callers stay byte-identical. - if allowed_paths is not None: - skipped_ids: set = set() - written_ids: set = set() - for fpath, result in by_file.items(): - target = skipped_ids if group_skipped(fpath) else written_ids - for n in result["nodes"]: - nid = n.get("id") - if nid is None: - continue - try: - hash(nid) - except TypeError: - continue - target.add(nid) - # A duplicate-attribution node (defined in a skipped AND a written - # group) still reaches the cache — don't over-prune references to it. - skipped_ids -= written_ids - if skipped_ids: - - def edge_dangles(e: dict) -> bool: - try: - return e.get("source") in skipped_ids or e.get("target") in skipped_ids - except TypeError: - # Non-hashable endpoint from an untrusted result; leave it - # to build-time validation rather than fail the save. - return False - - def hyperedge_dangles(h: dict) -> bool: - try: - return bool(skipped_ids & set(h.get("nodes") or [])) - except TypeError: - return False - - for fpath, result in by_file.items(): - if group_skipped(fpath): - continue - result["edges"] = [e for e in result["edges"] if not edge_dangles(e)] - result["hyperedges"] = [ - h for h in result["hyperedges"] if not hyperedge_dangles(h) - ] - - saved = 0 - skipped_not_file = 0 - for fpath, result in by_file.items(): - p = resolved_source_path(fpath) - if p.is_file(): - if allowed_paths is not None and p not in allowed_paths: - warnings.warn( - "semantic cache skipped out-of-scope source_file " - f"{fpath!r}; the file was not dispatched for extraction", - RuntimeWarning, - stacklevel=2, - ) + allowed = { + _relative(Path(item) if Path(item).is_absolute() else Path(root) / item, Path(root)) + for item in allowed_source_files or [] + } + partial = { + _relative( + Path(item) if Path(item).is_absolute() else Path(root) / item, + Path(root), + ) + for item in partial_source_files or [] + } + grouped: dict[str, dict[str, list[dict]]] = {} + for bucket, items in (("nodes", nodes), ("edges", edges), ("hyperedges", hyperedges or [])): + for item in items: + if not isinstance(item, dict) or not item.get("source_file"): continue - if merge_existing: - # allow_legacy=False: merging a pre-fingerprint entry into this - # write would fuse two prompt vintages inside a single entry and - # then stamp the result as current-vintage — the exact mixing - # #1939 is about, made unfixable because the entry now claims a - # prompt that only produced half of it. - # allow_partial=True: a file split into slices across chunks - # accumulates here; if an earlier slice truncated, keep its nodes - # in the union AND let the entry stay partial (the _partial - # markers ride through, so is_partial below re-detects it) rather - # than a later clean slice silently replacing it and promoting the - # half-file to complete. - prev = load_cached(p, root, kind=kind, cache_root=cache_root, - prompt=prompt, prompt_file=prompt_file, - allow_legacy=False, allow_partial=True) - _prev_partial = bool(prev.get("partial")) if prev else False - if prev: - result = { - "nodes": (prev.get("nodes", []) or []) + result["nodes"], - "edges": (prev.get("edges", []) or []) + result["edges"], - "hyperedges": (prev.get("hyperedges", []) or []) + result["hyperedges"], - } - else: - _prev_partial = False - # A file is partial if the caller named it, any of its grouped items - # carries the intrinsic ``_partial`` marker, OR the entry it merged - # onto was already partial (an empty-parse truncation leaves a - # ``partial: True`` entry with no item markers, so a later clean slice - # merging over it must NOT silently promote the half-file to complete - # — #1950). Copy so the caller's dict is never mutated. A genuine - # complete re-extraction (merge_existing=False) overwrites the - # content-hash key with a non-partial entry that then serves normally. - is_partial = ( - (partial_paths is not None and p in partial_paths) - or _group_has_partial_marker(result) - or _prev_partial + source = str(item["source_file"]).replace("\\", "/") + absolute = Path(source) if Path(source).is_absolute() else Path(root) / source + relative = _relative(absolute, Path(root)) + if allowed and relative not in allowed: + continue + grouped.setdefault(relative, {"nodes": [], "edges": [], "hyperedges": []})[bucket].append(item) + for relative in partial: + if not allowed or relative in allowed: + grouped.setdefault(relative, {"nodes": [], "edges": [], "hyperedges": []}) + saved = 0 + for relative, result in grouped.items(): + path = Path(root) / relative + if not path.is_file(): + continue + key = _cache_key(path, Path(root), kind, prompt, prompt_file) + previous_partial = bool(cache.get(key, {}).get("partial")) + if merge_existing: + previous = load_cached( + path, root, kind=kind, prompt=prompt, prompt_file=prompt_file, + allow_partial=True, cache=cache, ) - if is_partial: - result = {**result, "partial": True} - save_cached(p, result, root, kind=kind, cache_root=cache_root, - prompt=prompt, prompt_file=prompt_file) - saved += 1 - else: - skipped_not_file += 1 - if skipped_not_file and skipped_not_file == len(by_file): - warnings.warn( - f"save_semantic_cache: all {skipped_not_file} source_file group(s) were " - "skipped because their paths do not resolve to real files. This usually " - "means ``root`` is anchored to the wrong directory (e.g. the --out " - "directory instead of the corpus root). Pass the corpus directory as " - "``root`` and the output directory as ``cache_root`` (#1991).", - RuntimeWarning, - stacklevel=2, + if previous: + for bucket in ("nodes", "edges", "hyperedges"): + result[bucket] = [*previous.get(bucket, []), *result[bucket]] + if relative in partial: + for bucket in result.values(): + for item in bucket: + item["_partial"] = True + save_cached( + path, result, root, kind=kind, prompt=prompt, + prompt_file=prompt_file, cache=cache, + partial=( + relative in partial + or _group_has_partial_marker(result) + or (merge_existing and previous_partial) + ), ) + saved += 1 return saved + + +def prune_semantic_cache( + cache: dict[str, dict[str, Any]], live_hashes: set[str] +) -> int: + doomed = [ + key for key, entry in cache.items() + if key.startswith("semantic") and entry.get("content_hash") not in live_hashes + ] + for key in doomed: + del cache[key] + return len(doomed) + + +__all__ = [ + "_body_content", "_group_has_partial_marker", "cached_files", "cached_word_count", "check_semantic_cache", + "clear_cache", "file_hash", "load_cached", "prompt_fingerprint", + "prune_semantic_cache", "save_cached", "save_semantic_cache", +] diff --git a/graphify/callflow_html.py b/graphify/callflow_html.py index 181e7493b..ec1f2eeb8 100644 --- a/graphify/callflow_html.py +++ b/graphify/callflow_html.py @@ -2,7 +2,7 @@ """ callflow_html.py — Generate call-flow architecture HTML from graphify knowledge graph outputs. -Reads graph.json plus optional GRAPH_REPORT.md, .graphify_labels.json, and sections JSON, +Reads an embedded Helix generation plus optional report, labels, and sections, then produces a self-contained HTML file with: - Dark-themed CSS (fixed template) - Navigation bar from section list @@ -13,8 +13,8 @@ Usage: python3 -m graphify export callflow-html - python3 -m graphify export callflow-html /path/to/project/graphify-out/graph.json - python3 -m graphify export callflow-html --graph /path/to/graph.json --output docs/architecture.html + python3 -m graphify export callflow-html /path/to/project + python3 -m graphify export callflow-html --graph /path/to/graph.helix --output docs/architecture.html """ from __future__ import annotations @@ -136,7 +136,7 @@ def endpoint_id(value) -> str: def normalize_node(raw: dict, index: int) -> dict: - """Normalize a graphify node across common graph.json schema variants.""" + """Normalize a Graphify node for the presentation model.""" node = dict(raw) node_id = first_present( node, @@ -219,61 +219,47 @@ def normalize_edge(raw: dict, index: int) -> dict | None: return edge -def _node_link_payload(data: dict) -> tuple[list, list] | None: - """Read current graphify graph.json via NetworkX's node-link parser.""" - if not isinstance(data.get("nodes"), list): - return None - if not isinstance(data.get("links"), list) and not isinstance(data.get("edges"), list): - return None - - try: - from networkx.readwrite import json_graph - - try: - graph = json_graph.node_link_graph(data, edges="links") - except TypeError: - graph = json_graph.node_link_graph(data) - except Exception: - return None - - nodes = [] - for node_id, attrs in graph.nodes(data=True): - node = dict(attrs) - node["id"] = node_id - nodes.append(node) - - edges = [] - for index, (source, target, attrs) in enumerate(graph.edges(data=True), 1): - edge = dict(attrs) - edge["source"] = edge.get("_src", edge.get("source", source)) - edge["target"] = edge.get("_tgt", edge.get("target", target)) - edge.setdefault("id", f"edge_{index}") - edges.append(edge) - return nodes, edges - - def load_graph(path: str | Path) -> tuple: - """Load graph.json. Returns normalized (nodes, edges, hyperedges, metadata).""" - if path: - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(Path(path)) - except ValueError as exc: - raise SystemExit(f"ERROR: {exc}") from exc - data = read_json(path) - if not isinstance(data, dict): - raise SystemExit(f"ERROR: graph file must contain a JSON object: {path}") - - graph_block = data.get("graph") if isinstance(data.get("graph"), dict) else {} - meta_block = data.get("metadata") if isinstance(data.get("metadata"), dict) else {} - - node_link = _node_link_payload(data) - if node_link: - raw_nodes, raw_edges = node_link - else: - raw_nodes = first_list(data.get("nodes"), data.get("vertices"), graph_block.get("nodes"), graph_block.get("vertices")) - raw_edges = first_list(data.get("links"), data.get("edges"), graph_block.get("links"), graph_block.get("edges")) - hyperedges = first_list(data.get("hyperedges"), graph_block.get("hyperedges"), data.get("groups"), graph_block.get("groups")) + """Load one native generation into the call-flow presentation records.""" + from graphify.helix.model import edge_attributes, graphify_attributes + from graphify.helix.persistence import load_graph as load_helix_graph + from graphify.security import validate_store_path + + loaded = load_helix_graph(validate_store_path(path)) + graph = loaded.graph + community_records = [ + record for record in loaded.state.get("communities", []) + if isinstance(record, dict) and isinstance(record.get("id"), int) + ] + membership = { + member: record["id"] + for record in community_records + for member in record.get("members", []) + } + community_names = {record["id"]: record.get("name") for record in community_records} + raw_nodes = [] + for node in graph.nodes(): + attrs = graphify_attributes(node.attributes) + cid = membership.get(node.id) + raw_nodes.append({ + "id": node.id, + **attrs, + "community": cid, + "community_name": community_names.get(cid), + }) + raw_edges = [ + { + "id": str(edge.id), + "source": edge.source, + "target": edge.target, + **edge_attributes(edge), + } + for edge in graph.edges() + ] + graph_block = dict(graph.attributes).get("graph", {}) + graph_block = graph_block if isinstance(graph_block, dict) else {} + hyperedges = list(graph_block.get("hyperedges", [])) + meta_block = dict(loaded.metadata) nodes = [normalize_node(n, i) for i, n in enumerate(raw_nodes) if isinstance(n, dict)] edges = [] @@ -286,33 +272,23 @@ def load_graph(path: str | Path) -> tuple: meta = dict(graph_block) meta.update(meta_block) + build_meta = loaded.state.get("build", {}) + build_meta = build_meta if isinstance(build_meta, dict) else {} for key in ("built_at_commit", "commit", "project_name", "repo", "repository", "language_breakdown"): - if data.get(key) and not meta.get(key): - meta[key] = data.get(key) + if build_meta.get(key) and not meta.get(key): + meta[key] = build_meta.get(key) if meta.get("commit") and not meta.get("built_at_commit"): meta["built_at_commit"] = meta["commit"] + meta["community_labels"] = { + str(record["id"]): str(record.get("name") or f"Community {record['id']}") + for record in community_records + } + return nodes, edges, hyperedges, meta -def load_labels(path: str | Path | None) -> dict: - """Load community labels from .graphify_labels.json, tolerating wrapper keys.""" - data = read_json(path, default={}) - if not isinstance(data, dict): - return {} - if isinstance(data.get("labels"), dict): - data = data["labels"] - if isinstance(data.get("communities"), dict): - data = data["communities"] - labels = {} - for key, value in data.items(): - if isinstance(value, dict): - value = first_present(value, "label", "name", "title", default=key) - labels[str(key)] = str(value) - return labels - - -def load_sections(path: str | Path | None) -> list: +def load_sections(path: str | Path) -> list: """Load section definitions from JSON file.""" data = read_json(path, default=[]) if isinstance(data, dict) and isinstance(data.get("sections"), list): @@ -418,22 +394,20 @@ def resolve_graphify_paths(args) -> dict: graphify_out = Path(args.graphify_out).expanduser() elif args.graph: graphify_out = Path(args.graph).expanduser().parent - elif (base / "graph.json").exists(): + elif (base / "graph.helix").is_dir(): graphify_out = base else: graphify_out = base / GRAPHIFY_OUT project_root = graphify_out.parent if graphify_out.name == GRAPHIFY_OUT_NAME else base - graph = Path(args.graph).expanduser() if args.graph else graphify_out / "graph.json" + graph = Path(args.graph).expanduser() if args.graph else graphify_out / "graph.helix" report = Path(args.report).expanduser() if args.report else graphify_out / "GRAPH_REPORT.md" - labels = Path(args.labels).expanduser() if args.labels else graphify_out / ".graphify_labels.json" sections = Path(args.sections).expanduser() if args.sections else None return { "base": project_root, "graphify_out": graphify_out, "graph": graph, "report": report, - "labels": labels, "sections": sections, } @@ -1518,7 +1492,6 @@ def __init__( graphify_out: str | Path | None = None, graph: str | Path | None = None, report: str | Path | None = None, - labels: str | Path | None = None, sections: str | Path | None = None, output: str | Path | None = None, lang: str = "auto", @@ -1531,7 +1504,6 @@ def __init__( self.graphify_out = str(graphify_out) if graphify_out is not None else None self.graph = str(graph) if graph is not None else None self.report = str(report) if report is not None else None - self.labels = str(labels) if labels is not None else None self.sections = str(sections) if sections is not None else None self.output = str(output) if output is not None else None self.lang = lang @@ -1582,7 +1554,6 @@ def write_callflow_html( graphify_out: str | Path | None = None, graph: str | Path | None = None, report: str | Path | None = None, - labels: str | Path | None = None, sections: str | Path | None = None, output: str | Path | None = None, lang: str = "auto", @@ -1598,7 +1569,6 @@ def write_callflow_html( graphify_out=graphify_out, graph=graph, report=report, - labels=labels, sections=sections, output=output, lang=lang, @@ -1612,12 +1582,12 @@ def write_callflow_html( if not paths["graph"].exists(): raise FileNotFoundError( f"graphify output not found: {paths['graph']}. " - "Run graphify first or pass --graph /path/to/graph.json." + "Run graphify first or pass --graph /path/to/graph.helix." ) # Load data nodes, edges, hyperedges, meta = load_graph(paths["graph"]) - labels = load_labels(paths["labels"]) + labels = dict(meta.get("community_labels", {})) lang = detect_lang(args.lang, nodes, labels) if paths["sections"]: sections = load_sections(paths["sections"]) @@ -1627,7 +1597,7 @@ def write_callflow_html( report_text = load_report(paths["report"]) if not nodes: - raise ValueError("graph.json contains 0 nodes") + raise ValueError("the active Helix generation contains 0 nodes") if len(sections) <= 1: raise ValueError("no sections defined") @@ -1637,7 +1607,7 @@ def write_callflow_html( node_ids = {node.get("id") for node in nodes} missing_endpoint_edges = [edge for edge in edges if edge.get("source") not in node_ids or edge.get("target") not in node_ids] if verbose and missing_endpoint_edges: - print(f"WARNING: {len(missing_endpoint_edges)} edges reference nodes not present in graph.json.", file=sys.stderr) + print(f"WARNING: {len(missing_endpoint_edges)} edges reference missing nodes.", file=sys.stderr) meta["project_name"] = infer_project_name(str(paths["graph"]), meta) meta["node_count"] = len(nodes) @@ -1985,9 +1955,8 @@ def main(): ) parser.add_argument("project", nargs="?", default=None, help="Project root or graphify output directory") parser.add_argument("--graphify-out", default=None, help="Path to graphify output directory") - parser.add_argument("--graph", default=None, help="Path to graph.json") + parser.add_argument("--graph", default=None, help="Path to graph.helix store") parser.add_argument("--report", default=None, help="Path to GRAPH_REPORT.md") - parser.add_argument("--labels", default=None, help="Path to .graphify_labels.json") parser.add_argument("--sections", default=None, help="Path to sections JSON file; auto-derived when omitted") parser.add_argument("--output", default=None, help="Output HTML path") parser.add_argument("--lang", default="auto", help="HTML language: auto, zh-CN, en, etc. (default: auto)") @@ -2003,7 +1972,6 @@ def main(): graphify_out=args.graphify_out, graph=args.graph, report=args.report, - labels=args.labels, sections=args.sections, output=args.output, lang=args.lang, diff --git a/graphify/cli.py b/graphify/cli.py index 35e397e60..8aa3b0a6d 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1,27 +1,24 @@ -"""graphify command dispatch — every non-install subcommand. +"""Helix-only command dispatch for every non-install Graphify command.""" -Extracted verbatim from __main__.main(); __main__ now calls dispatch_command(cmd) -after the install/platform dispatch. Kept out of __main__ to shrink the CLI entry -module. The path-redirect (`graphify ` -> extract) re-enters via a lazy -import of main to avoid a cli<->__main__ import cycle. -""" from __future__ import annotations + import json import os import re import sys import time -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT from pathlib import Path +from typing import Any + +from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT _SEARCH_NUDGE = json.dumps({ "hookSpecificOutput": { "hookEventName": "PreToolUse", "additionalContext": ( - 'MANDATORY: graphify-out/graph.json exists. You MUST run ' - '`graphify query ""` before grepping raw files. Only grep ' - 'after graphify has oriented you, or to modify/debug specific lines.' + "MANDATORY: graphify-out/graph.helix exists. Run " + "`graphify query \"\"` before broad source searches." ), } }, ensure_ascii=False, separators=(",", ":")) + "\n" @@ -29,13 +26,8 @@ "hookSpecificOutput": { "hookEventName": "PreToolUse", "additionalContext": ( - 'MANDATORY: graphify-out/graph.json exists. You MUST run graphify ' - 'before reading source files. Use: `graphify query ""` ' - '(scoped subgraph), `graphify explain ""`, or ' - '`graphify path "" ""`. Only read raw files after graphify has ' - 'oriented you, or to modify/debug specific lines. This rule applies to ' - 'subagents too — include it in every subagent prompt involving code ' - 'exploration.' + "MANDATORY: orient with graphify-out/graph.helix before broad source reads. " + "Use `graphify query`, `graphify explain`, or `graphify path`." ), } }, ensure_ascii=False, separators=(",", ":")) + "\n" @@ -43,267 +35,39 @@ "hookSpecificOutput": { "hookEventName": "PreToolUse", "additionalContext": ( - 'graphify-out/graph.json exists but may be STALE for this file (the file ' - 'changed after the last build). Prefer `graphify query ""` for ' - 'orientation, and run `graphify update` to refresh the graph. Reading the ' - 'file directly is fine.' + "graphify-out/graph.helix may be stale for this file. Use `graphify query` " + "for orientation and run `graphify update`; reading the file is allowed." ), } }, ensure_ascii=False, separators=(",", ":")) + "\n" -# Strict-mode block (opt-in). Claude Code PreToolUse honors -# hookSpecificOutput.permissionDecision == "deny" and shows permissionDecisionReason -# to the model. Fires at most once per session (see _mark_session_denied) so it can -# never strand an agent: the very next read proceeds with the soft nudge. _READ_DENY = json.dumps({ "hookSpecificOutput": { "hookEventName": "PreToolUse", "permissionDecision": "deny", "permissionDecisionReason": ( - 'graphify strict mode: this project has a fresh knowledge graph that covers ' - 'this file. Run `graphify query ""` (or `graphify explain` / ' - '`graphify path`) FIRST to orient yourself, then re-issue this Read — it ' - 'will be allowed. This block fires at most once per session; reading raw ' - 'files to modify or debug specific lines is fine after one query. Apply the ' - 'same rule in any subagent prompt that explores code.' + "graphify strict mode: run `graphify query `, `graphify explain`, " + "or `graphify path` first, then retry this read. This blocks at most once " + "per session." ), } }, ensure_ascii=False, separators=(",", ":")) + "\n" _HOOK_SOURCE_EXTS = ( - '.py', '.js', '.cjs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', - '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', - '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', + ".py", ".js", ".cjs", ".ts", ".tsx", ".jsx", ".astro", ".vue", ".svelte", + ".go", ".rs", ".java", ".rb", ".c", ".h", ".cpp", ".hpp", ".cc", + ".cs", ".kt", ".swift", ".php", ".scala", ".lua", ".sh", ".md", + ".rst", ".txt", ".mdx", ) _GEMINI_NUDGE_TEXT = ( - 'graphify: knowledge graph at graphify-out/. For focused questions, run ' - '`graphify query ""` (scoped subgraph, usually much smaller than ' - 'GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only ' - 'for broad architecture context.' + "graphify: native knowledge graph at graphify-out/graph.helix. " + "Use `graphify query \"\"` before broad source searches." ) -def _default_graph_path() -> str: - return str(Path(_GRAPHIFY_OUT) / "graph.json") - - -def _stamped_manifest_files( - files_by_type: dict[str, list[str]], - sem_result: dict, - root: Path, - partial_source_files: "set[str] | None" = None, -) -> dict[str, list[str]]: - """Manifest-safe files dict: only stamp semantic files that actually - produced output (cache hit or fresh extraction). Files whose chunk failed - have no source_file entry in sem_result — leaving their semantic_hash - empty so detect_incremental re-queues them (#933). - - A file in ``partial_source_files`` DID produce output this run, but only a - truncated fragment of it, so it is excluded from stamping too — otherwise - detect_incremental would see it "done" and never re-dispatch it, leaving the - incomplete node set live forever on the warm-incremental path. Same #933 - mechanism: leave it unstamped and it is re-queued next run. - - Both sides of the membership test are resolved against the scan ``root`` - before comparing (#1897): node/edge/hyperedge ``source_file`` values are - root-relative on a fresh extraction while ``files_by_type`` entries are - absolute (from detect()), so a raw string comparison never matched and - every freshly-extracted semantic doc was dropped from the manifest. - Mirrors the #1890 path normalization in graphify.llm. - - Hyperedges are counted as output (#1920): a chunk whose only result for a - document is a hyperedge (3+ nodes sharing a concept) is valid output that - the semantic cache persists per-``source_file`` — omitting it here left the - doc unstamped, so detect_incremental re-queued it on every run. The stamping - condition mirrors the cache-write keying (a hyperedge carries its own - ``source_file``); do not derive it from member nodes. - """ - root = Path(root) - - def _resolve(value: str) -> Path: - p = Path(value) - if not p.is_absolute(): - p = root / p - try: - return p.resolve() - except (OSError, RuntimeError): - return p - - sem_extracted: set[Path] = set() - for coll in ("nodes", "edges", "hyperedges"): - for item in sem_result.get(coll, []): - sf = item.get("source_file", "") - if sf: - sem_extracted.add(_resolve(sf)) - partial_resolved = {_resolve(p) for p in (partial_source_files or set())} - sem_types = {"document", "paper", "image"} - return { - ftype: [ - f for f in flist - if ftype not in sem_types - or (_resolve(f) in sem_extracted and _resolve(f) not in partial_resolved) - ] - for ftype, flist in files_by_type.items() - } - - -def _stale_graph_sources( - graph_path: Path, - scan_root: Path, - seen_files: set[str], -) -> list[str]: - """Source files graph.json still references but the current scan no longer - contains (#1909). - - Incremental extract's prune set was historically derived from the manifest - alone (``manifest - corpus``), so a file that became EXCLUDED - (.graphifyignore/.gitignore/--exclude changed) without being listed in the - manifest kept its stale nodes in graph.json forever. Derive prune - candidates from the graph's own node ``source_file``s instead: anything - the graph references that the post-exclude detect corpus no longer - contains is stale, whether the file was deleted or newly excluded. - - Only IN-ROOT paths are candidates: out-of-root/absolute entries - (--include sources, symlinked external corpora) are never walked by - detect, so their absence from the corpus is not staleness evidence. - Relative entries are re-anchored against both the scan root and the - graph's own output root; only anchors that land inside the scan root - count. Since #1941 extracts always store source_file relative to the SCAN - root, so the scan-root anchor is the live one; the out-root anchor stays - for graphs written by <=0.9.16, which stored them relative to the OUT root - (e.g. ``../project/x.py``, #555/#1899). - ``seen_files`` must be the FULL detect output including unclassified - files, so nodes from walked-but-unsupported sources (e.g. introspected - Cargo.toml manifests) are not misread as stale. - """ - try: - data = json.loads(graph_path.read_text(encoding="utf-8")) - except Exception: - return [] - if not isinstance(data, dict): - return [] - try: - root_res = scan_root.resolve() - except (OSError, RuntimeError): - root_res = scan_root - # /graphify-out/graph.json — relative source_files may be anchored here. - out_base = graph_path.parent.parent - try: - out_base = out_base.resolve() - except (OSError, RuntimeError): - pass - - def _within_root(p: Path) -> bool: - try: - p.relative_to(root_res) - return True - except ValueError: - pass - try: - p.resolve().relative_to(root_res) - return True - except (ValueError, OSError, RuntimeError): - return False - - def _in_seen(p: Path) -> bool: - if str(p) in seen_files: - return True - try: - return str(p.resolve()) in seen_files - except (OSError, RuntimeError): - return False - - stale: list[str] = [] - checked: set[str] = set() - for n in data.get("nodes", []): - if not isinstance(n, dict): - continue - sf = n.get("source_file") - if not sf or not isinstance(sf, str) or sf in checked: - continue - checked.add(sf) - if "://" in sf: - continue # remote/virtual source (e.g. Google Workspace), not a scanned path - p = Path(sf) - if p.is_absolute(): - candidates = [p] - else: - rel = sf.replace("\\", "/") - bases = [root_res] - if out_base != root_res: - bases.append(out_base) - candidates = [ - Path(os.path.normpath(str(base / rel))) for base in bases - ] - in_root = [c for c in candidates if _within_root(c)] - if not in_root: - continue # out-of-root under every anchor: never prune - if any(_in_seen(c) for c in in_root): - continue # still part of the scan corpus - stale.append(sf) - return stale - - -def _prune_graph_json_sources(graph_path: Path, stale_sources: list[str]) -> int: - """Drop nodes/edges/hyperedges owned by ``stale_sources`` from graph.json - in place. Returns the number of nodes removed. - - Used by the ``--no-cluster`` incremental early-exit: that path never runs - ``build_merge`` (it would raw-dump only the new chunks), so an - exclusion-only change must prune the existing raw graph directly or the - newly-excluded file's nodes survive forever (#1909). - ``stale_sources`` comes from :func:`_stale_graph_sources`, i.e. the - graph's own ``source_file`` spellings, so exact string matching is enough. - """ - try: - data = json.loads(graph_path.read_text(encoding="utf-8")) - except Exception: - return 0 - if not isinstance(data, dict): - return 0 - stale = set(stale_sources) - links_key = "links" if "links" in data else "edges" - nodes = [n for n in data.get("nodes", []) if isinstance(n, dict)] - kept_nodes = [n for n in nodes if n.get("source_file") not in stale] - removed_ids = { - n.get("id") for n in nodes if n.get("source_file") in stale - } - n_removed = len(nodes) - len(kept_nodes) - kept_edges = [ - e for e in data.get(links_key, []) - if isinstance(e, dict) - and e.get("source_file") not in stale - and e.get("source") not in removed_ids - and e.get("target") not in removed_ids - ] - kept_hyper = [ - h for h in data.get("hyperedges", []) - if isinstance(h, dict) and h.get("source_file") not in stale - ] - if n_removed == 0 and len(kept_edges) == len(data.get(links_key, [])) and ( - len(kept_hyper) == len(data.get("hyperedges", [])) - ): - return 0 - data["nodes"] = kept_nodes - data[links_key] = kept_edges - if "hyperedges" in data: - data["hyperedges"] = kept_hyper - from graphify.export import backup_if_protected as _backup - _backup(graph_path.parent) - from graphify.paths import write_json_atomic - write_json_atomic(graph_path, data, indent=2) - return n_removed - - class _StageTimer: - """Print per-stage wall-clock timings to stderr when --timing is set (#1490). - - Monotonic (perf_counter), diagnostic-only: emits ``[graphify timing] : - N.Ns`` after each stage and a final total. Off by default, so normal output is - byte-identical and machine-read stdout is untouched. - """ - def __init__(self, enabled: bool) -> None: - import time as _time - self._now = _time.perf_counter + import time + + self._now = time.perf_counter self.enabled = enabled self.start = self._now() self._last = self.start @@ -317,41 +81,59 @@ def mark(self, stage: str) -> None: def total(self) -> None: if self.enabled: print(f"[graphify timing] total: {self._now() - self.start:.1f}s", file=sys.stderr) -def _enforce_graph_size_cap_or_exit(gp: Path) -> None: - """Reject oversized graph files before parsing (CLI exit-on-fail flavor). - - Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the - raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. - Use this from ``__main__.py`` subcommands that already use the ``print + - sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, - ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, - ``global_graph``, ``watch``, ``export``) call the security helper directly - and let the ``ValueError`` propagate. - """ - from graphify.security import check_graph_file_size_cap + + +def _default_graph_path() -> str: + return str(Path(_GRAPHIFY_OUT) / "graph.helix") + + +def _option(args: list[str], *names: str) -> str | None: + for index, value in enumerate(args): + for name in names: + if value == name and index + 1 < len(args): + return args[index + 1] + if value.startswith(name + "="): + return value.split("=", 1)[1] + return None + + +def _store_arg(args: list[str], *, default: str | Path | None = None) -> Path: + value = _option(args, "--store", "--graph") or str( + default or Path(_GRAPHIFY_OUT) / "graph.helix" + ) + path = Path(value).expanduser() + if path.suffix.lower() == ".json" or path.is_file(): + raise ValueError( + "legacy JSON graphs are obsolete; pass a graph.helix store and rebuild from source" + ) + return path + + +def _validate_store_or_exit(gp: Path) -> None: + """Validate a native store path for command-line callers.""" + from graphify.security import validate_store_path + try: - check_graph_file_size_cap(gp) - except ValueError as exc: + validate_store_path(gp) + except (ValueError, FileNotFoundError) as exc: print(f"error: {exc}", file=sys.stderr) - sys.exit(1) + raise SystemExit(1) from exc + + def _hook_strict_enabled(flag: bool) -> bool: - """Resolve strict mode: GRAPHIFY_HOOK_STRICT env overrides the baked-in flag - (truthy forces on without a reinstall, falsy is the kill switch); unset defers - to the flag the installed hook command carried.""" - v = os.environ.get("GRAPHIFY_HOOK_STRICT", "").strip().lower() - if v in ("1", "true", "yes", "on"): + value = os.environ.get("GRAPHIFY_HOOK_STRICT", "").strip().lower() + if value in {"1", "true", "yes", "on"}: return True - if v in ("0", "false", "no", "off"): + if value in {"0", "false", "no", "off"}: return False return flag -def _touch_query_stamp(graph_path: "Path") -> None: - """Record that graphify oriented the agent recently, next to the queried graph. - The strict guard suppresses its block while this stamp is fresh. Fail-silent.""" +def _touch_query_stamp(graph_path: Path) -> None: try: from graphify.paths import write_text_atomic - stamp = Path(graph_path).parent / "cache" / "last_query_stamp" + + stamp = graph_path.parent / "cache" / "last_query_stamp" stamp.parent.mkdir(parents=True, exist_ok=True) write_text_atomic(stamp, str(time.time())) except Exception: @@ -359,3172 +141,736 @@ def _touch_query_stamp(graph_path: "Path") -> None: def _query_stamp_fresh() -> bool: - """True if a query/explain/path ran within GRAPHIFY_HOOK_STRICT_TTL (default - 1800s) — recent orientation, so strict mode does not block this read.""" from graphify.paths import out_path + try: ttl = float(os.environ.get("GRAPHIFY_HOOK_STRICT_TTL", "1800")) - return (time.time() - out_path("cache", "last_query_stamp").stat().st_mtime) < ttl + return time.time() - out_path("cache", "last_query_stamp").stat().st_mtime < ttl except Exception: return False def _mark_session_denied(session_id: str) -> bool: - """Atomically claim a one-time strict block for this session. Returns True only - on the FIRST call for a given session id (O_EXCL create wins once); every later - call — or any error — returns False, so a session is blocked at most once and an - agent can never be stranded. Best-effort GC of markers older than 24h.""" from graphify.paths import out_path - sid = re.sub(r"[^A-Za-z0-9_-]", "_", str(session_id))[:64] - if not sid: + + safe_id = re.sub(r"[^A-Za-z0-9_-]", "_", str(session_id))[:64] + if not safe_id: return False try: - d = out_path("cache", "hook_sessions") - d.mkdir(parents=True, exist_ok=True) - fd = os.open(str(d / f"{sid}.denied"), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + directory = out_path("cache", "hook_sessions") + directory.mkdir(parents=True, exist_ok=True) + fd = os.open( + str(directory / f"{safe_id}.denied"), + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + 0o644, + ) os.close(fd) - try: - cutoff = time.time() - 86400 - for entry in os.scandir(d): - try: - if entry.stat().st_mtime < cutoff: - os.unlink(entry.path) - except OSError: - pass - except OSError: - pass return True - except FileExistsError: + except (FileExistsError, OSError): return False + + +def _target_is_indexed(file_path: str, root: Path) -> bool: + from graphify.paths import out_path + + if not file_path: + return True + try: + manifest_path = out_path("manifest.json") + if manifest_path.stat().st_size > 2_000_000: + return True + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict) or not manifest: + return True + path = Path(file_path) + relative: set[str] = {path.name} + try: + relative.add(path.resolve().relative_to(root).as_posix()) + except (ValueError, OSError, RuntimeError): + pass + keys = {str(key).replace("\\", "/") for key in manifest} + absolute = str(path).replace("\\", "/") + return absolute in keys or any( + value and any(key == value or key.endswith("/" + value) for key in keys) + for value in relative + ) except Exception: - return False + return True def _run_hook_guard(kind: str, strict: bool = False) -> None: - """Shell-agnostic PreToolUse guard (#522). - - Reads the tool-call JSON from stdin and, when a fresh in-project knowledge graph - exists, nudges the agent to use graphify instead of grepping/reading raw files. - Replaces the old inline bash hooks that failed to parse on Windows. - - Fails open everywhere: any error, or a non-matching tool call, prints nothing - and the caller exits 0, so a legitimate tool call is never blocked by a bug. - - In strict mode (opt-in, Claude Code Read only) the FIRST raw read of indexed, - in-project, fresh code per session is DENIED with a redirect to `graphify query` - (permissionDecision), then downgrades to the soft nudge — it fires at most once - per session and can never strand the agent. Search (Bash) and Glob stay - nudge-only: a compound shell command has no single parseable target and blocking - file listing would strand navigation. #1840: reads of out-of-project files are - ignored, and a graph that is stale for the target file softens to a non-mandatory - nudge instead of blocking or demanding. - """ - from graphify.paths import out_path, GRAPHIFY_OUT_NAME - # Gemini's BeforeTool hook takes no stdin and must ALWAYS return a decision so - # the tool is never blocked; the graph nudge is appended only when a graph - # exists. Handled before the stdin read below (which the search/read guards need). + from graphify.paths import GRAPHIFY_OUT_NAME, out_path + if kind == "gemini": - payload = {"decision": "allow"} + payload: dict[str, Any] = {"decision": "allow"} try: - if out_path("graph.json").is_file(): + if out_path("graph.helix").is_dir(): payload["additionalContext"] = _GEMINI_NUDGE_TEXT except Exception: pass sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":"))) return try: - d = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) - except Exception: - return - if not isinstance(d, dict): - return - t = d.get("tool_input", d) - if not isinstance(t, dict): - return - try: + data = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) + if not isinstance(data, dict): + return + tool_input = data.get("tool_input", data) + if not isinstance(tool_input, dict): + return + store_path = out_path("graph.helix") + if not store_path.is_dir(): + return if kind == "search": - cmd_str = str(t.get("command", "") or "") - # Two input shapes reach this guard (matcher "Bash|Grep", #1986): - # the Bash tool carries `command`, while Claude Code's dedicated - # Grep tool carries `pattern` (plus optional path/glob) and no - # command — a Grep call IS a content search by definition, so it - # nudges whenever a graph exists. For Bash, keep matching the same - # set the old `case` matched: *grep*, *ripgrep*, and rg/find/fd/ - # ack/ag as a token (name followed by a space). Nudge-only, even in - # strict mode — see the docstring. - is_grep_tool = not cmd_str and bool(t.get("pattern")) - is_bash_search = any(tok in cmd_str for tok in ( - "grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ")) - if (is_grep_tool or is_bash_search) and out_path("graph.json").is_file(): + command = str(tool_input.get("command", "")) + is_grep_tool = not command and bool(tool_input.get("pattern")) + is_bash_search = any( + token in command + for token in ("grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ") + ) + if is_grep_tool or is_bash_search: sys.stdout.write(_SEARCH_NUDGE) - elif kind == "read": - vals = [str(t.get("file_path") or ""), str(t.get("pattern") or ""), str(t.get("path") or "")] - j = " ".join(vals).lower().replace("\\", "/") - tails = [ - "." + seg.rsplit(".", 1)[-1] - for v in vals if v - for seg in [v.lower().replace("\\", "/").rsplit("/", 1)[-1]] - if "." in seg - ] - under_out = "graphify-out/" in j or (GRAPHIFY_OUT_NAME.lower() + "/") in j - if under_out or not any(tl in _HOOK_SOURCE_EXTS for tl in tails): + return + if kind != "read": + return + + values = [ + str(tool_input.get("file_path") or ""), + str(tool_input.get("pattern") or ""), + str(tool_input.get("path") or ""), + ] + joined = " ".join(values).lower().replace("\\", "/") + tails = [Path(value).suffix.lower() for value in values if value] + if GRAPHIFY_OUT_NAME.lower() + "/" in joined or not any( + extension in _HOOK_SOURCE_EXTS for extension in tails + ): + return + + root = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) + try: + root = root.resolve() + except (OSError, RuntimeError): + pass + explicit = [ + str(tool_input.get(name) or "") + for name in ("file_path", "path") + if tool_input.get(name) + ] + if explicit: + in_project = False + for value in explicit: + path = Path(value) + if not path.is_absolute(): + in_project = True + break + try: + path.resolve().relative_to(root) + in_project = True + break + except (ValueError, OSError, RuntimeError): + continue + if not in_project: return - # #1840 (a): skip files outside the graph's project. cwd (or - # CLAUDE_PROJECT_DIR, which Claude Code sets) is the project root, since - # the guard only triggers when graph.json exists relative to cwd. A path - # candidate that resolves outside that root is out-of-project. - root = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) - try: - root = root.resolve() - except (OSError, RuntimeError): - pass - path_vals = [str(t.get("file_path") or ""), str(t.get("path") or "")] - explicit = [v for v in path_vals if v] - if explicit: - in_project = False - for v in explicit: - p = Path(v) - if not p.is_absolute(): - in_project = True # relative -> anchored at cwd == in project - break - try: - p.resolve().relative_to(root) - in_project = True - break - except (ValueError, OSError, RuntimeError): - continue - if not in_project: - return - # One stat for existence + mtime of the graph. + + graph_mtime = store_path.stat().st_mtime + file_path = str(tool_input.get("file_path") or "") + stale = False + if file_path: try: - gmtime = os.stat(str(out_path("graph.json"))).st_mtime + stale = Path(file_path).stat().st_mtime > graph_mtime except OSError: - return - # #1840 (b): stale-for-target -> soften, never block. The target file - # changed after the last build, or watch flagged the tree. - stale = False - fp = str(t.get("file_path") or "") - if fp: - try: - stale = os.stat(fp).st_mtime > gmtime - except OSError: - stale = False - try: - if out_path("needs_update").exists(): - stale = True - except Exception: pass - if stale: - sys.stdout.write(_READ_NUDGE_STALE) - return - # Strict block: Read tool only, first time per session, not recently - # oriented, and the file is demonstrably indexed. - tool_name = d.get("tool_name") - if _hook_strict_enabled(strict) and tool_name in (None, "Read") \ - and not _query_stamp_fresh() \ - and _target_is_indexed(fp, root) \ - and _mark_session_denied(str(d.get("session_id") or "")): - sys.stdout.write(_READ_DENY) - return - sys.stdout.write(_READ_NUDGE) + if out_path("needs_update").exists(): + stale = True + if stale: + sys.stdout.write(_READ_NUDGE_STALE) + return + if ( + _hook_strict_enabled(strict) + and data.get("tool_name") in (None, "Read") + and not _query_stamp_fresh() + and _target_is_indexed(file_path, root) + and _mark_session_denied(str(data.get("session_id") or "")) + ): + sys.stdout.write(_READ_DENY) + return + sys.stdout.write(_READ_NUDGE) except Exception: - pass + return -def _target_is_indexed(file_path: str, root: "Path") -> bool: - """Guard the strict deny: only block a read of a file the graph actually indexes. - Reads manifest.json (cheap, capped); on any doubt (missing/corrupt/oversized - manifest, unresolvable path) returns True so the once-per-session deny still - applies — that block is self-limiting, so erring toward it is safe.""" - from graphify.paths import out_path - if not file_path: - return True - try: - mp = out_path("manifest.json") - st = mp.stat() - if st.st_size > 2_000_000: - return True - manifest = json.loads(mp.read_text(encoding="utf-8")) - if not isinstance(manifest, dict) or not manifest: - return True - p = Path(file_path) - rels = set() - try: - rels.add(p.resolve().relative_to(root).as_posix()) - except (ValueError, OSError, RuntimeError): - pass - rels.add(p.name) - keys = {str(k).replace("\\", "/") for k in manifest} - abskey = str(p).replace("\\", "/") - return abskey in keys or any(r and (r in keys or any(k.endswith("/" + r) or k == r for k in keys)) for r in rels) - except Exception: - return True -def _clone_repo( - url: str, branch: str | None = None, out_dir: Path | None = None -) -> Path: - """Clone a GitHub repo to a local cache dir and return the path. - - Clones into ~/.graphify/repos// by default so repeated - runs on the same URL reuse the existing clone (git pull instead of clone). - """ - import subprocess as _sp - import re as _re - - # Normalise URL — strip trailing .git if present - url = url.rstrip("/") - if not url.endswith(".git"): - git_url = url + ".git" - else: - git_url = url - url = url[:-4] - - # Extract owner/repo from URL - m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) - if not m: - print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) - sys.exit(1) - owner, repo = m.group(1), m.group(2) - - if out_dir: - dest = out_dir - else: - dest = Path.home() / ".graphify" / "repos" / owner / repo +def _clone_repo(url: str, branch: str | None = None, out_dir: Path | None = None) -> Path: + import re + import subprocess + clean = url.rstrip("/") + git_url = clean if clean.endswith(".git") else clean + ".git" + match = re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", clean) + if not match: + raise ValueError(f"not a recognized GitHub URL: {url}") + owner, repo = match.group(1), match.group(2) + destination = out_dir or Path.home() / ".graphify" / "repos" / owner / repo if branch and branch.startswith("-"): - print(f"error: invalid branch name: {branch!r}", file=sys.stderr) - sys.exit(1) - - if dest.exists(): - print(f"Repo already cloned at {dest} - pulling latest...", flush=True) - cmd = ["git", "-C", str(dest), "pull"] + raise ValueError(f"invalid branch name: {branch!r}") + if destination.exists(): + command = ["git", "-C", str(destination), "pull"] if branch: - cmd += ["origin", "--", branch] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) + command += ["origin", "--", branch] else: - dest.parent.mkdir(parents=True, exist_ok=True) - print(f"Cloning {url} -> {dest} ...", flush=True) - cmd = ["git", "clone", "--depth", "1"] + destination.parent.mkdir(parents=True, exist_ok=True) + command = ["git", "clone", "--depth", "1"] if branch: - cmd += ["--branch", branch] - cmd += ["--", git_url, str(dest)] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) - sys.exit(1) + command += ["--branch", branch] + command += ["--", git_url, str(destination)] + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode: + raise RuntimeError(result.stderr.strip() or "git command failed") + print(f"Ready at: {destination}") + return destination - print(f"Ready at: {dest}", flush=True) - return dest +def _loaded(args: list[str]): + from graphify.helix.persistence import load_graph + from graphify.security import validate_store_path -def _reenter_main() -> None: - from graphify.__main__ import main - main() + return load_graph(validate_store_path(_store_arg(args))) -def dispatch_command(cmd: str) -> None: - if cmd == "provider": - from graphify.llm import _custom_providers_path, BACKENDS - import json as _json - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - global_path = _custom_providers_path(global_=True) - - if subcmd == "list": - global_path.parent.mkdir(parents=True, exist_ok=True) - existing: dict = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if not existing: - print("No custom providers registered.") - else: - for name in existing: - print(f" {name} ({existing[name].get('base_url', '')})") - - elif subcmd == "show": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider show ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - print(_json.dumps({name: existing[name]}, indent=2)) - - elif subcmd == "add": - args = sys.argv[3:] - name = args[0] if args and not args[0].startswith("-") else "" - if not name: - print("Usage: graphify provider add --base-url URL --default-model MODEL --env-key KEY", file=sys.stderr) - sys.exit(1) - if name in BACKENDS: - print(f"Error: '{name}' is a built-in provider and cannot be overridden.", file=sys.stderr) - sys.exit(1) - base_url = "" - default_model = "" - env_key = "" - pricing_input = 0.0 - pricing_output = 0.0 - i = 1 - while i < len(args): - a = args[i] - if a == "--base-url" and i + 1 < len(args): - base_url = args[i + 1]; i += 2 - elif a.startswith("--base-url="): - base_url = a.split("=", 1)[1]; i += 1 - elif a == "--default-model" and i + 1 < len(args): - default_model = args[i + 1]; i += 2 - elif a.startswith("--default-model="): - default_model = a.split("=", 1)[1]; i += 1 - elif a == "--env-key" and i + 1 < len(args): - env_key = args[i + 1]; i += 2 - elif a.startswith("--env-key="): - env_key = a.split("=", 1)[1]; i += 1 - elif a == "--pricing-input" and i + 1 < len(args): - pricing_input = float(args[i + 1]); i += 2 - elif a == "--pricing-output" and i + 1 < len(args): - pricing_output = float(args[i + 1]); i += 2 - else: - i += 1 - if not base_url or not default_model or not env_key: - print("Error: --base-url, --default-model, and --env-key are required.", file=sys.stderr) - sys.exit(1) - from graphify.llm import provider_base_url_ok - if not provider_base_url_ok(base_url, name): - print(f"Error: refusing to add provider with unsafe base_url {base_url!r}.", file=sys.stderr) - sys.exit(1) - global_path.parent.mkdir(parents=True, exist_ok=True) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - existing[name] = { - "base_url": base_url, - "default_model": default_model, - "env_key": env_key, - "pricing": {"input": pricing_input, "output": pricing_output}, - "temperature": 0, - } - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' added. Use with: graphify extract . --backend {name}") - - elif subcmd == "remove": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider remove ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - del existing[name] - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' removed.") - - else: - print("Usage: graphify provider [add|list|show|remove]", file=sys.stderr) - if subcmd: - sys.exit(1) - elif cmd == "prs": - from graphify.prs import cmd_prs - cmd_prs(sys.argv[2:]) - elif cmd == "hook": - from graphify.hooks import ( - install as hook_install, - uninstall as hook_uninstall, - status as hook_status, +def _communities(loaded) -> tuple[dict[int, list[Any]], dict[int, str], dict[int, float]]: + communities: dict[int, list[Any]] = {} + labels: dict[int, str] = {} + cohesion: dict[int, float] = {} + for record in loaded.state.get("communities", []): + if not isinstance(record, dict) or not isinstance(record.get("id"), int): + continue + cid = record["id"] + communities[cid] = list(record.get("members", [])) + labels[cid] = str(record.get("name") or f"Community {cid}") + if isinstance(record.get("cohesion"), (int, float)): + cohesion[cid] = float(record["cohesion"]) + return communities, labels, cohesion + + +def _query(args: list[str]) -> None: + from graphify.serve import _query_graph_text + + question = args[0] if args and not args[0].startswith("-") else "" + if not question: + raise ValueError("query requires a question") + loaded = _loaded(args) + _touch_query_stamp(loaded.store_path) + budget = int(_option(args, "--budget") or 2000) + mode = "dfs" if "--dfs" in args else "bfs" + filters: list[str] = [] + for index, value in enumerate(args): + if value == "--context" and index + 1 < len(args): + filters.append(args[index + 1]) + elif value.startswith("--context="): + filters.append(value.split("=", 1)[1]) + print(_query_graph_text( + loaded.graph, + question, + native_query=loaded.query, + mode=mode, + depth=2, + token_budget=budget, + context_filters=filters, + learning_overlay=( + dict(learning.get("nodes", {})) + if isinstance((learning := loaded.state.get("learning", {})), dict) + else {} + ), + )) + + +def _path(args: list[str]) -> None: + from graphify.helix.model import edge_attributes, node_attributes + from graphify.serve import _pick_scored_endpoint, _query_terms, _score_nodes + + positional = [value for value in args if not value.startswith("-")] + if len(positional) < 2: + raise ValueError("path requires source and target") + loaded = _loaded(args) + _touch_query_stamp(loaded.store_path) + graph = loaded.graph + source_scores = _score_nodes( + graph, _query_terms(positional[0]), native_query=loaded.query + ) + target_scores = _score_nodes( + graph, _query_terms(positional[1]), native_query=loaded.query + ) + if not source_scores or not target_scores: + raise ValueError("source or target did not resolve uniquely") + source = _pick_scored_endpoint(graph, source_scores, positional[0]) + target = _pick_scored_endpoint(graph, target_scores, positional[1]) + result = graph.shortest_path(source, target, direction="both") + node_ids = getattr(result, "node_ids", ()) + if not node_ids: + print("No path found.") + raise SystemExit(0) + path = list(node_ids) + segments = [] + for index, (left, right) in enumerate(zip(path, path[1:])): + records = [ + graph.edge(edge_id) + for edge_id in graph.incident_edge_ids(left) + ] + edge = next( + ( + record for record in records + if record is not None + and {record.source, record.target} == {left, right} + ), + None, ) - - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - print(hook_install(Path("."))) - elif subcmd == "uninstall": - print(hook_uninstall(Path("."))) - elif subcmd == "status": - print(hook_status(Path("."))) + if edge is None: + raise RuntimeError("native shortest path returned an edge-less hop") + attrs = edge_attributes(edge) + relation = attrs.get("relation", "related") + confidence = attrs.get("confidence") + confidence_text = f" [{confidence}]" if confidence else "" + if index == 0: + segments.append(str(node_attributes(graph, left).get("label", left))) + label = str(node_attributes(graph, right).get("label", right)) + if edge.source == left and edge.target == right: + segments.append(f"--{relation}{confidence_text}--> {label}") else: - print("Usage: graphify hook [install|uninstall|status]", file=sys.stderr) - sys.exit(1) - elif cmd == "query": - if len(sys.argv) < 3: - print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.serve import _query_graph_text - from graphify.security import sanitize_label - from networkx.readwrite import json_graph - from graphify import querylog - - question = sys.argv[2] - use_dfs = "--dfs" in sys.argv - budget = 2000 - graph_path = _default_graph_path() - context_filters: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--budget" and i + 1 < len(args): - try: - budget = int(args[i + 1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--budget="): - try: - budget = int(args[i].split("=", 1)[1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--context" and i + 1 < len(args): - context_filters.append(args[i + 1]) - i += 2 - elif args[i].startswith("--context="): - context_filters.append(args[i].split("=", 1)[1]) - i += 1 - elif args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print(f"error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - try: - import json as _json - import networkx as _nx - - _raw = _json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - try: - from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(_raw.get("nodes", [])): - print( - "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " - "rebuild with `graphify extract --force` to get path-qualified IDs " - "(fixes same-name-file collisions).", - file=sys.stderr, - ) - except Exception: - pass - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - import time as _time - _t0 = _time.perf_counter() - _mode = "dfs" if use_dfs else "bfs" - _result = _query_graph_text( - G, - question, - mode=_mode, - depth=2, - token_budget=budget, - context_filters=context_filters, - ) - querylog.log_query( - kind="query", - question=question, - corpus=str(gp), - result=_result, - mode=_mode, - depth=2, - token_budget=budget, - duration_ms=(_time.perf_counter() - _t0) * 1000, - ) - _touch_query_stamp(gp) - print(_result) - elif cmd == "affected": - if len(sys.argv) < 3: - print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph - query = sys.argv[2] - graph_path = _default_graph_path() - depth = 2 - relations: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - elif args[i].startswith("--graph="): - graph_path = args[i].split("=", 1)[1] - i += 1 - elif args[i] == "--depth" and i + 1 < len(args): - try: - depth = int(args[i + 1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--depth="): - try: - depth = int(args[i].split("=", 1)[1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--relation" and i + 1 < len(args): - relations.append(args[i + 1]) - i += 2 - elif args[i].startswith("--relation="): - relations.append(args[i].split("=", 1)[1]) - i += 1 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - try: - graph = load_graph(gp) - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - print( - format_affected( - graph, - query, - relations=relations or DEFAULT_AFFECTED_RELATIONS, - depth=depth, + segments.append(f"<--{relation}{confidence_text}-- {label}") + print(f"Shortest path ({len(path) - 1} hops):\n " + " ".join(segments)) + + +def _explain(args: list[str]) -> None: + from graphify.helix.model import edge_attributes, node_attributes + from graphify.serve import _find_node + + query = next((value for value in args if not value.startswith("-")), "") + if not query: + raise ValueError("explain requires a node label or ID") + loaded = _loaded(args) + _touch_query_stamp(loaded.store_path) + graph = loaded.graph + matches = _find_node(graph, query, native_query=loaded.query) + if len(matches) > 1: + query_path = query.replace("\\", "/") + file_matches = [ + node_id for node_id in matches + if str(node_attributes(graph, node_id).get("source_file", "")).replace("\\", "/") == query_path + and str(node_attributes(graph, node_id).get("label", "")) == Path(query_path).name + ] + if len(file_matches) == 1: + matches = file_matches + if len(matches) != 1: + print(f"No unique node match for {query}") + return + node_id = matches[0] + attrs = node_attributes(graph, node_id) + print(f"Node: {attrs.get('label', node_id)}") + print(f"ID: {node_id}") + print(f"Source: {attrs.get('source_file', '-')} {attrs.get('source_location', '')}".rstrip()) + print(f"Type: {attrs.get('file_type', '-')}" ) + print(f"Degree: {graph.degree(node_id).degree}") + learning = loaded.state.get("learning", {}) + entries = learning.get("nodes", {}) if isinstance(learning, dict) else {} + entry = entries.get(str(node_id)) if isinstance(entries, dict) else None + if isinstance(entry, dict) and entry.get("status"): + status = entry["status"] + if status == "preferred": + lesson = ( + f"Lesson: preferred source (start here) — {entry.get('uses', 0)} useful, " + f"score={entry.get('score', 0)}" + ) + elif status == "contested": + lesson = ( + f"Lesson: contested (useful {entry.get('uses', 0)} / " + f"dead-end {entry.get('neg', 0)})" ) - ) - elif cmd == "save-result": - # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] - # [--outcome useful|dead_end|corrected] [--correction TEXT] - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify save-result") - p.add_argument("--question", required=True) - p.add_argument("--answer", default=None) - p.add_argument("--answer-file", dest="answer_file", default=None) - p.add_argument("--type", dest="query_type", default="query") - p.add_argument("--nodes", nargs="*", default=[]) - p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) - p.add_argument("--correction", default=None) - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - opts = p.parse_args(sys.argv[2:]) - if opts.answer_file: - opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() - elif not opts.answer: - p.error("--answer or --answer-file is required") - from graphify.ingest import save_query_result as _sqr - - out = _sqr( - question=opts.question, - answer=opts.answer, - memory_dir=Path(opts.memory_dir), - query_type=opts.query_type, - source_nodes=opts.nodes or None, - outcome=opts.outcome, - correction=opts.correction, - ) - print(f"Saved to {out}") - elif cmd == "reflect": - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify reflect") - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - p.add_argument( - "--out", - default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), - ) - p.add_argument("--graph", default=None) - p.add_argument("--analysis", default=None) - p.add_argument("--labels", default=None) - p.add_argument("--half-life-days", type=float, default=30.0, - help="signal weight halves every N days (default 30)") - p.add_argument("--min-corroboration", type=int, default=2, - help="distinct useful results to promote a node to preferred (default 2)") - p.add_argument("--if-stale", action="store_true", - help="skip when LESSONS.md is already newer than every input " - "(e.g. the git hook just refreshed it)") - opts = p.parse_args(sys.argv[2:]) - from graphify.reflect import reflect as _reflect, lessons_fresh as _lessons_fresh - - graph_arg = opts.graph - if graph_arg is None: - default_graph = Path(_GRAPHIFY_OUT) / "graph.json" - if default_graph.exists(): - graph_arg = str(default_graph) - - _gp = Path(graph_arg) if graph_arg else None - _analysis_path = None - _labels_path = None - if _gp is not None: - _analysis_path = Path(opts.analysis) if opts.analysis else ( - _gp.parent / ".graphify_analysis.json") - _labels_path = Path(opts.labels) if opts.labels else ( - _gp.parent / ".graphify_labels.json") - - if opts.if_stale and _lessons_fresh( - Path(opts.out), Path(opts.memory_dir), _gp, _analysis_path, _labels_path - ): - print(f"Lessons already up to date -> {opts.out} (skipped; omit --if-stale to force)") else: - out_path, agg = _reflect( - memory_dir=Path(opts.memory_dir), - out_path=Path(opts.out), - graph_path=_gp, - analysis_path=_analysis_path, - labels_path=_labels_path, - half_life_days=opts.half_life_days, - min_corroboration=opts.min_corroboration, - ) - c = agg["counts"] - print( - f"Reflected {agg['total']} memories " - f"({c['useful']} useful, {c['dead_end']} dead ends, " - f"{c['corrected']} corrected) -> {out_path}" - ) - elif cmd == "path": - if len(sys.argv) < 4: - print( - 'Usage: graphify path "" "" [--graph path]', - file=sys.stderr, - ) - sys.exit(1) - from graphify.serve import _pick_scored_endpoint, _score_nodes - from networkx.readwrite import json_graph - import networkx as _nx - - source_label = sys.argv[2] - target_label = sys.argv[3] - graph_path = _default_graph_path() - args = sys.argv[4:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) - tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) - if not src_scored: - print(f"No node matching '{source_label}' found.", file=sys.stderr) - sys.exit(1) - if not tgt_scored: - print(f"No node matching '{target_label}' found.", file=sys.stderr) - sys.exit(1) - src_nid = _pick_scored_endpoint(G, src_scored, source_label) - tgt_nid = _pick_scored_endpoint(G, tgt_scored, target_label) - # Ambiguity guard: when both queries resolve to the same node, the - # shortest path is trivially zero hops, which is almost never what the - # caller wanted (see bug #828). - if src_nid == tgt_nid: - print( - f"'{source_label}' and '{target_label}' both resolved to the same " - f"node '{src_nid}'. Use a more specific label or the exact node ID.", - file=sys.stderr, - ) - sys.exit(1) - for _name, _scored, _nid in ( - ("source", src_scored, src_nid), - ("target", tgt_scored, tgt_nid), - ): - # A close runner-up only made the resolution ambiguous when the raw - # score head is what got picked; a full-token override was chosen on - # token coverage, not score, so the head's margin is irrelevant. - if len(_scored) >= 2 and _nid == _scored[0][1]: - _top, _runner = _scored[0][0], _scored[1][0] - if _top > 0 and (_top - _runner) / _top < 0.10: - print( - f"warning: {_name} match was ambiguous " - f"(top score {_top:g}, runner-up {_runner:g})", - file=sys.stderr, - ) - try: - path_nodes = _nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) - except (_nx.NetworkXNoPath, _nx.NodeNotFound): - print(f"No path found between '{source_label}' and '{target_label}'.") - sys.exit(0) - hops = len(path_nodes) - 1 - segments = [] - from graphify.build import edge_data - for i in range(len(path_nodes) - 1): - u, v = path_nodes[i], path_nodes[i + 1] - # Check which direction the stored edge points. - if G.has_edge(u, v): - edata = edge_data(G, u, v) - forward = True - else: - edata = edge_data(G, v, u) - forward = False - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - conf_str = f" [{conf}]" if conf else "" - if i == 0: - segments.append(G.nodes[u].get("label", u)) - if forward: - segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") - else: - segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") - print(f"Shortest path ({hops} hops):\n " + " ".join(segments)) - from graphify import querylog - querylog.log_query( - kind="path", - question=f"{sys.argv[2]} -> {sys.argv[3]}", - corpus=str(gp), - nodes_returned=hops, - ) - _touch_query_stamp(gp) - - elif cmd == "explain": - if len(sys.argv) < 3: - print('Usage: graphify explain "" [--graph path]', file=sys.stderr) - sys.exit(1) - from graphify.serve import _find_node - from networkx.readwrite import json_graph - - label = sys.argv[2] - graph_path = _default_graph_path() - args = sys.argv[3:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - matches = _find_node(G, label) - if not matches: - print(f"No node matching '{label}' found.") - sys.exit(0) - nid = matches[0] - d = G.nodes[nid] - print(f"Node: {d.get('label', nid)}") - print(f" ID: {nid}") - print( - f" Source: {d.get('source_file', '')} {d.get('source_location', '')}".rstrip() - ) - print(f" Type: {d.get('file_type', '')}") - print(f" Community: {d.get('community_name') or d.get('community', '')}") - # Work-memory overlay: a derived experiential hint from `graphify reflect`, - # merged in display-only from the .graphify_learning.json sidecar next to - # graph.json. No line when the node has no overlay entry. - try: - from graphify.reflect import load_learning_overlay as _llo - from graphify.security import sanitize_label as _sl - _overlay = _llo(gp) - _entry = _overlay.get(str(nid)) - if _entry: - _status = _sl(str(_entry.get("status", ""))) - if _status == "contested": - _line = (f" Lesson: contested (useful {_entry.get('uses', 0)} / " - f"dead-end {_entry.get('neg', 0)})") - elif _status == "preferred": - _line = (f" Lesson: preferred source (start here) — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - else: - _line = (f" Lesson: {_status or 'tentative'} — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - if _entry.get("stale"): - _line += " [code changed since — re-verify]" - print(_line) - except Exception: - pass - print(f" Degree: {G.degree(nid)}") - from graphify.build import edge_data - connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) - for nb in G.successors(nid): - connections.append(("out", nb, edge_data(G, nid, nb))) - for nb in G.predecessors(nid): - connections.append(("in", nb, edge_data(G, nb, nid))) - if connections: - print(f"\nConnections ({len(connections)}):") - connections.sort(key=lambda c: G.degree(c[1]), reverse=True) - for direction, nb, edata in connections[:20]: - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - arrow = "-->" if direction == "out" else "<--" - print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]") - if len(connections) > 20: - print(f" ... and {len(connections) - 20} more") - from graphify import querylog - querylog.log_query( - kind="explain", - question=sys.argv[2], - corpus=str(gp), - nodes_returned=len(connections), + lesson = f"Lesson: {status} ({entry.get('uses', 0)} useful)" + print(lesson) + print("Connections:") + for edge_id in graph.incident_edge_ids(node_id): + edge = graph.edge(edge_id) + if edge is None: + continue + neighbor = edge.target if edge.source == node_id else edge.source + relation = edge_attributes(edge).get("relation", "related") + arrow = "-->" if edge.source == node_id else "<--" + print(f" {arrow} {node_attributes(graph, neighbor).get('label', neighbor)} [{relation}]") + + +def _export(args: list[str]) -> None: + if not args: + raise ValueError("export requires a format") + kind = args[0] + tail = args[1:] + positional_store = tail[0] if tail and not tail[0].startswith("-") else None + store_args = ["--store", positional_store, *tail[1:]] if positional_store else tail + store_path = _store_arg(store_args) + from graphify.helix.persistence import load_graph + from graphify.security import validate_store_path + + loaded = load_graph(validate_store_path(store_path)) + graph = loaded.graph + communities, labels, cohesion = _communities(loaded) + output = _option(args, "--out", "--output") + from graphify import export + + if kind == "html": + html_output = output or str(Path(_GRAPHIFY_OUT) / "graph.html") + if "--no-viz" in args: + Path(html_output).unlink(missing_ok=True) + else: + export.to_html(graph, communities, html_output, community_labels=labels) + elif kind == "graphml": + export.to_graphml(graph, communities, output or str(Path(_GRAPHIFY_OUT) / "graph.graphml")) + elif kind in {"cypher", "neo4j", "falkordb"}: + export.to_cypher(graph, output or str(Path(_GRAPHIFY_OUT) / "cypher.txt")) + elif kind == "svg": + export.to_svg(graph, communities, output or str(Path(_GRAPHIFY_OUT) / "graph.svg"), community_labels=labels) + elif kind == "obsidian": + count = export.to_obsidian( + graph, communities, output or str(Path(_GRAPHIFY_OUT) / "obsidian"), + community_labels=labels, cohesion=cohesion, ) - _touch_query_stamp(gp) - - elif cmd == "diagnose": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd != "multigraph": - print( - "Usage: graphify diagnose multigraph " - "[--graph path] [--json] [--max-examples N] " - "[--directed] [--undirected] [--extract-path path]", - file=sys.stderr, - ) - sys.exit(1) - - graph_path = Path(_default_graph_path()) - max_examples = 5 - directed: bool | None = None - direction_flag: str | None = None - json_output = False - extract_path: Path | None = None - - i = 3 - while i < len(sys.argv): - arg = sys.argv[i] - if arg == "--graph": - i += 1 - if i >= len(sys.argv): - print("error: --graph requires a path", file=sys.stderr) - sys.exit(1) - graph_path = Path(sys.argv[i]) - elif arg == "--json": - json_output = True - elif arg == "--max-examples": - i += 1 - if i >= len(sys.argv): - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - try: - max_examples = int(sys.argv[i]) - except ValueError: - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - if max_examples < 0: - print("error: --max-examples must be >= 0", file=sys.stderr) - sys.exit(1) - elif arg == "--directed": - if direction_flag == "undirected": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "directed" - directed = True - elif arg == "--undirected": - if direction_flag == "directed": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "undirected" - directed = False - elif arg == "--extract-path": - i += 1 - if i >= len(sys.argv): - print("error: --extract-path requires a path", file=sys.stderr) - sys.exit(1) - extract_path = Path(sys.argv[i]) - else: - print(f"error: unknown diagnose option {arg}", file=sys.stderr) - sys.exit(1) - i += 1 - - from graphify.diagnostics import ( - diagnose_file, - format_diagnostic_json, - format_diagnostic_report, + print(f"Wrote {count} Obsidian notes.") + elif kind == "canvas": + export.to_canvas(graph, communities, output or str(Path(_GRAPHIFY_OUT) / "graph.canvas"), community_labels=labels) + elif kind == "wiki": + from graphify.wiki import to_wiki + + count = to_wiki( + graph, communities, output or str(Path(_GRAPHIFY_OUT) / "wiki"), + community_labels=labels, cohesion=cohesion, + god_nodes_data=list(loaded.state.get("analysis", {}).get("god_nodes", [])), ) + print(f"Wrote {count} wiki articles.") + elif kind == "callflow-html": + from graphify.callflow_html import write_callflow_html - try: - summary = diagnose_file( - graph_path, - directed=directed, - root=Path(".").resolve(), - max_examples=max_examples, - extract_path=extract_path, - ) - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) + result = write_callflow_html(graph=store_path, output=output, verbose=True) + print(result) + else: + raise ValueError(f"unknown export format: {kind}") - if json_output: - print(json.dumps(format_diagnostic_json(summary), indent=2)) - else: - print(format_diagnostic_report(summary)) - elif cmd == "add": - if len(sys.argv) < 3: - print( - "Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", - file=sys.stderr, - ) - sys.exit(1) - from graphify.ingest import ingest as _ingest - - url = sys.argv[2] - author: str | None = None - contributor: str | None = None - target_dir = Path("raw") - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--author" and i + 1 < len(args): - author = args[i + 1] - i += 2 - elif args[i] == "--contributor" and i + 1 < len(args): - contributor = args[i + 1] - i += 2 - elif args[i] == "--dir" and i + 1 < len(args): - target_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - try: - saved = _ingest(url, target_dir, author=author, contributor=contributor) - print(f"Saved to {saved}") - print("Run /graphify --update in your AI assistant to update the graph.") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd == "watch": - watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import watch as _watch +def _global(args: list[str]) -> None: + from graphify.global_graph import global_add, global_list, global_path, global_remove - try: - _watch(watch_path) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd in ("cluster-only", "label"): - # `label` is `cluster-only` that always (re)generates community names with - # the configured backend, even when a .graphify_labels.json already exists. - force_relabel = cmd == "label" - # Mirror the tree/export arg-parsing pattern: walk argv so flags and - # the optional positional path can appear in any order (#724). - no_viz = "--no-viz" in sys.argv - no_label = "--no-label" in sys.argv - missing_only = "--missing-only" in sys.argv - co_timing = "--timing" in sys.argv - _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) - label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None - _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) - label_model = _model_arg.split("=", 1)[1] if _model_arg else None - _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) - min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 - args = sys.argv[2:] - watch_path: Path | None = None - graph_override: Path | None = None - co_resolution: float = 1.0 - co_exclude_hubs: float | None = None - label_max_concurrency: int = 4 - label_batch_size: int = 100 - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_override = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--backend" and i_arg + 1 < len(args): - label_backend = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--backend="): - label_backend = a.split("=", 1)[1]; i_arg += 1 - elif a == "--model" and i_arg + 1 < len(args): - label_model = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--model="): - label_model = a.split("=", 1)[1]; i_arg += 1 - elif a == "--resolution" and i_arg + 1 < len(args): - co_resolution = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--resolution="): - co_resolution = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--exclude-hubs" and i_arg + 1 < len(args): - co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--exclude-hubs="): - co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--max-concurrency" and i_arg + 1 < len(args): - label_max_concurrency = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--max-concurrency="): - label_max_concurrency = int(a.split("=", 1)[1]); i_arg += 1 - elif a == "--batch-size" and i_arg + 1 < len(args): - label_batch_size = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--batch-size="): - label_batch_size = int(a.split("=", 1)[1]); i_arg += 1 - elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): - i_arg += 1 - elif a.startswith("--"): - i_arg += 1 - elif watch_path is None: - watch_path = Path(a); i_arg += 1 - else: - i_arg += 1 - if watch_path is None: - watch_path = Path(".") - graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" - if not graph_json.exists(): - print( - f"error: no graph found at {graph_json} — run /graphify first", - file=sys.stderr, - ) - sys.exit(1) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json - from graphify.cluster import cluster, score_all, remap_communities_to_previous - from graphify.analyze import ( - god_nodes, - surprising_connections, - suggest_questions, - ) - from graphify.report import generate - from graphify.export import to_json, to_html - - stages = _StageTimer(co_timing) - print("Loading existing graph...") - # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. - # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the - # graph.html render below falls back to the community-aggregation view - # (node_limit=5000) when over the cap. - from graphify.security import check_graph_file_size_cap as _check_cap - _over_cap = False - try: - _check_cap(graph_json) - except ValueError: - _over_cap = True - try: - _over_cap_bytes = graph_json.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) - _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) - print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") - stages.mark("load") - print("Re-clustering...") - communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) - # Mirror the watch/update path (#822): map new cids to prior ones by - # node-overlap so the existing .graphify_labels.json keeps attaching - # to the same conceptual community after re-clustering. Without this, - # labels follow raw cid index and become misaligned whenever the - # graph has changed between labeling and cluster-only (#1027). - previous_node_community = { - n["id"]: n["community"] - for n in _raw.get("nodes", []) - if n.get("community") is not None and n.get("id") is not None - } - if previous_node_community: - communities = remap_communities_to_previous(communities, previous_node_community) - stages.mark("cluster") - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - stages.mark("analyze") - # Where outputs (GRAPH_REPORT.md, re-clustered graph.json, labels, - # analysis, html) land. When `--graph` points at a graph INSIDE a - # graphify-out/ dir (another project/tenant's output), write beside it, - # not into a stray graphify-out/ in the CWD (#1747). But when `--graph` - # points at an arbitrary path — e.g. a `backup/graph.json` archived - # before re-clustering (#934) — fall back to the CWD's graphify-out/, - # which is the restore-into-place workflow that test pins. The default - # (no --graph) case already has graph_json under watch_path/graphify-out. - _out_name = Path(_GRAPHIFY_OUT).name - if graph_override is not None and graph_json.parent.name == _out_name: - out = graph_json.parent - else: - out = watch_path / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - labels_path = out / ".graphify_labels.json" - existing_labels: dict[int, str] = {} - if labels_path.exists(): - try: - existing_labels = { - int(k): v - for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - existing_labels = {} - # Accumulate token usage from the labeling LLM calls so cluster-only mode - # reports real cost instead of a hardcoded zero (#1694). Stays {0, 0} on - # the reuse / no-label paths, which make no LLM calls. - label_token_usage = {"input": 0, "output": 0} - if labels_path.exists() and not force_relabel: - # Reuse saved labels, but don't blindly trust them: the graph may have - # been re-scoped/re-clustered since labeling, in which case a cid now - # covers a DIFFERENT community and its old (LLM) name is wrong (#label-stale). - # Validate each community against the membership signature saved beside the - # labels; any community that changed (or has no saved label) is renamed by - # its current hub — deterministic and correct-by-construction — and the user - # is told to `graphify label` for fresh LLM names. Unchanged communities keep - # their saved label. When no signature sidecar exists (labels predate this), - # fall back to hub-filling only the communities missing a label. - from graphify.cluster import community_member_sigs, label_communities_by_hub - sig_path = labels_path.parent / (labels_path.name + ".sig") - saved_sigs: dict[int, str] = {} - if sig_path.exists(): - try: - saved_sigs = { - int(k): v for k, v in - json.loads(sig_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - saved_sigs = {} - cur_sigs = community_member_sigs(communities) - count_mismatch = len(existing_labels) != len(communities) - labels = {} - hub_labels: dict[int, str] | None = None - changed = 0 - for cid in communities: - have_label = cid in existing_labels - if saved_sigs: - # Precise: the membership signature tells us if this exact - # community changed since it was labeled. - fresh = have_label and saved_sigs.get(cid) == cur_sigs.get(cid) - else: - # No signature sidecar (labels predate it). A differing community - # COUNT means the labels describe a different clustering, so a cid's - # old label can't be trusted; equal count is the best "same" signal. - fresh = have_label and not count_mismatch - if fresh: - labels[cid] = existing_labels[cid] - else: - if hub_labels is None: - hub_labels = label_communities_by_hub(G, communities) - labels[cid] = hub_labels[cid] - if have_label: - changed += 1 - if changed: - print( - f"[graphify] community set changed since labeling " - f"({len(existing_labels)} saved labels, {len(communities)} communities now; " - f"renamed {changed} community(ies) by their hub). " - f"Run `graphify label` to refresh names with the LLM.", - file=sys.stderr, - ) - elif no_label and not force_relabel: - labels = {cid: f"Community {cid}" for cid in communities} - else: - # No labels file yet (or `graphify label` forced a refresh). When run - # standalone there is no orchestrating agent to do skill.md Step 5, so - # auto-name communities rather than leave "Community N" (#1097). - from graphify.cluster import label_communities_by_hub - from graphify.llm import generate_community_labels - print("Labeling communities...") - # Deterministic, LLM-free base labels: name each community after its - # highest-degree hub, so the report is readable even with no backend - # (previously bare "Community N"). A configured LLM backend overrides these - # with richer names below; its no-backend placeholder fallback does NOT. - hub_labels = label_communities_by_hub(G, communities) - label_communities_input = communities - labels = dict(hub_labels) - if missing_only: - labels = { - cid: existing_labels.get(cid, hub_labels[cid]) - for cid in communities - } - label_communities_input = { - cid: members - for cid, members in communities.items() - if cid not in existing_labels or existing_labels.get(cid) == f"Community {cid}" - } - generated_labels, _ = generate_community_labels( - G, label_communities_input, backend=label_backend, model=label_model, gods=gods, - max_concurrency=label_max_concurrency, batch_size=label_batch_size, - usage_out=label_token_usage, - ) - # Only let the LLM OVERRIDE where it produced a real name — its no-backend - # fallback returns "Community {cid}" placeholders, which must not clobber - # the deterministic hub labels. - labels.update({ - cid: v for cid, v in generated_labels.items() - if v and v != f"Community {cid}" - }) - stages.mark("label") - questions = suggest_questions(G, communities, labels) - tokens = label_token_usage - from graphify.export import _git_head as _gh - _commit = _gh() - from graphify.report import load_learning_for_report as _llfr - report = generate(G, communities, cohesion, labels, gods, surprises, - {"warning": "cluster-only mode — file stats not available"}, - tokens, str(watch_path), suggested_questions=questions, - min_community_size=min_community_size, built_at_commit=_commit, - learning=_llfr(out / "graph.json")) - (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") - stages.mark("report") - from graphify.export import backup_if_protected as _backup - _backup(out) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "questions": questions, - } - (out / ".graphify_analysis.json").write_text( - json.dumps(analysis, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - to_json(G, communities, str(out / "graph.json"), community_labels=labels) - from graphify.paths import write_json_atomic as _wja - _wja(labels_path, {str(k): v for k, v in labels.items()}, ensure_ascii=False) - # Membership signatures beside the labels so a later cluster-only can detect - # which communities changed and avoid reusing a stale label (see reuse above). - from graphify.cluster import community_member_sigs as _cms - (labels_path.parent / (labels_path.name + ".sig")).write_text( - json.dumps({str(k): v for k, v in _cms(communities).items()}), encoding="utf-8") - - # Mirror watch.py pattern: gate to_html so core outputs (graph.json + - # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise - # fall back to ValueError handling so an oversized graph doesn't crash - # the CLI mid-write and leave a stale graph.html on disk. - html_target = out / "graph.html" - if no_viz: - if html_target.exists(): - html_target.unlink() - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") - else: - try: - # Over-cap fallback (#1019): force the community-aggregation - # path so an oversized graph still renders a usable graph.html. - _node_limit = 5000 if _over_cap else None - to_html(G, communities, str(html_target), community_labels=labels or None, - node_limit=_node_limit) - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") - except ValueError as viz_err: - if html_target.exists(): - html_target.unlink() - print(f"Skipped graph.html: {viz_err}") - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") - - elif cmd == "update": - force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") - no_cluster = False - args = sys.argv[2:] - watch_arg: str | None = None - for a in args: - if a == "--force": - force = True - continue - if a == "--no-cluster": - no_cluster = True - continue - if a.startswith("-"): - print(f"error: unknown update option: {a}", file=sys.stderr) - sys.exit(2) - if watch_arg is not None: - print("error: update accepts at most one path argument", file=sys.stderr) - sys.exit(2) - watch_arg = a - - if watch_arg is not None: - watch_path = Path(watch_arg) - else: - # Try to recover the scan root saved by the last full build - saved = Path(_GRAPHIFY_OUT) / ".graphify_root" - if saved.exists(): - watch_path = Path(saved.read_text(encoding="utf-8").strip()) - else: - watch_path = Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import _rebuild_code - - print(f"Re-extracting code files in {watch_path} (no LLM needed)...") - # Interactive CLI: block on the per-repo lock rather than skip, so the - # user sees their explicit `graphify update` complete instead of - # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) - if ok: - print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") - if not ( - os.environ.get("GEMINI_API_KEY") - or os.environ.get("GOOGLE_API_KEY") - or os.environ.get("MOONSHOT_API_KEY") - or os.environ.get("DEEPSEEK_API_KEY") - or os.environ.get("GRAPHIFY_NO_TIPS") - ): - print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") - else: - print( - "Nothing to update or rebuild failed — check output above.", - file=sys.stderr, - ) - sys.exit(1) - - elif cmd == "hook-check": - # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. - # Keep this as a cross-platform no-op so installed hooks never break Bash - # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. - sys.exit(0) - elif cmd == "hook-guard": - # Shell-agnostic Claude/Codebuddy PreToolUse guard (#522). Replaces the old - # inline-bash hooks that failed on Windows. Prints an additionalContext nudge - # toward graphify when a fresh in-project graph exists; always exits 0. In - # strict mode (opt-in, `hook-guard read --strict`) it blocks the first raw - # read per session via the JSON permissionDecision payload — never via exit - # code — and downgrades to the nudge thereafter. - _run_hook_guard( - sys.argv[2] if len(sys.argv) > 2 else "", - strict="--strict" in sys.argv[3:], + if not args or args[0] == "list": + repos = global_list() + if not repos: + print("No projects in the global graph.") + for tag, record in sorted(repos.items()): + print(f"{tag}: {record.get('node_count', 0)} nodes ({record.get('source_path', '')})") + return + if args[0] == "add" and len(args) >= 2: + source = Path(args[1]) + tag = _option(args[2:], "--as") or source.resolve().name + print(global_add(source, tag, retain_rollback="--retain-rollback" in args)) + return + if args[0] == "remove" and len(args) >= 2: + print( + f"Removed {global_remove(args[1], retain_rollback='--retain-rollback' in args)} " + "nodes." ) - sys.exit(0) - elif cmd == "check-update": - if len(sys.argv) < 3: - print("Usage: graphify check-update ", file=sys.stderr) - sys.exit(1) - from graphify.watch import check_update - - check_update(Path(sys.argv[2]).resolve()) - sys.exit(0) - elif cmd == "tree": - # Emit a D3 v7 collapsible-tree HTML view of graph.json: - # expand-all / collapse-all / reset-view buttons, multi-line - # wrapText labels with separately-coloured name + count, - # depth-based palette, click-to-toggle subtree, hover inspector - # showing top-K outbound edges per symbol. - from typing import Optional as _Opt - from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - output_path: "_Opt[Path]" = None - root: "_Opt[str]" = None - max_children = DEFAULT_MAX_CHILDREN - top_k_edges = 0 - project_label: "_Opt[str]" = None - args = sys.argv[2:] - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--output" and i_arg + 1 < len(args): - output_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--root" and i_arg + 1 < len(args): - root = args[i_arg + 1]; i_arg += 2 - elif a == "--max-children" and i_arg + 1 < len(args): - max_children = int(args[i_arg + 1]); i_arg += 2 - elif a == "--top-k-edges" and i_arg + 1 < len(args): - top_k_edges = int(args[i_arg + 1]); i_arg += 2 - elif a == "--label" and i_arg + 1 < len(args): - project_label = args[i_arg + 1]; i_arg += 2 - elif a in ("-h", "--help"): - print("Usage: graphify tree [--graph PATH] [--output HTML]") - print(" --graph PATH path to graph.json (default graphify-out/graph.json)") - print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") - print(" --root PATH filesystem root (default: longest common dir of all source_files)") - print(" --max-children N cap visible children per node (default 200)") - print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") - print(" --label NAME project label shown in the page header") - return - else: - i_arg += 1 - if not graph_path.is_file(): - print(f"error: graph.json not found at {graph_path}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(graph_path) - if output_path is None: - output_path = graph_path.parent / "GRAPH_TREE.html" - out = write_tree_html( - graph_path=graph_path, output_path=output_path, - root=root, max_children=max_children, - top_k_edges=top_k_edges, project_label=project_label, + return + if args[0] == "path": + print(global_path()) + return + raise ValueError("usage: graphify global add [--as TAG] | remove TAG | list | path") + + +def _update_or_extract(cmd: str, args: list[str]) -> None: + from graphify.watch import _rebuild_code, _write_build_config + + target = Path(next((value for value in args if not value.startswith("-")), ".")) + force = "--force" in args + no_cluster = "--no-cluster" in args + changed: list[Path] | None = None + if cmd == "update" and _option(args, "--changed"): + changed = [Path(value) for value in str(_option(args, "--changed")).split(",")] + include_semantic = cmd == "extract" and "--code-only" not in args + output_value = _option(args, "--out") + output = Path(output_value) if output_value else None + if "--no-gitignore" in args: + config_root = output if output is not None else target + _write_build_config( + config_root / _GRAPHIFY_OUT, + excludes=None, + gitignore=False, ) - size_kb = out.stat().st_size / 1024 - print(f"wrote {out} ({size_kb:.1f} KB)") - print(f"open with: xdg-open {out} (or file://{out.resolve()})") - sys.exit(0) - - elif cmd == "merge-driver": - # git merge driver for graph.json — takes (base, current, other) and writes - # the union of current+other nodes/edges back to current. Exits 1 on - # corrupt input so git surfaces the conflict instead of silently - # accepting a poisoned merge (see F-005). - # Usage: graphify merge-driver %O %A %B (set in .git/config merge driver) - if len(sys.argv) < 5: - print("Usage: graphify merge-driver ", file=sys.stderr) - sys.exit(1) - _base_path, _current_path, _other_path = sys.argv[2], sys.argv[3], sys.argv[4] - # Hard caps so a malicious or corrupted graph.json cannot exhaust memory - # at parse time. 50 MB / 100k nodes are well above any realistic graph - # (typical graphs are <5 MB / <50k nodes); anything larger should fail - # the merge so a human can investigate. - _MERGE_MAX_BYTES = 50 * 1024 * 1024 - _MERGE_MAX_NODES = 100_000 - import networkx as _nx - from networkx.readwrite import json_graph as _jg - def _load_graph(p: str): - path_obj = Path(p) - try: - size = path_obj.stat().st_size - except OSError as exc: - raise RuntimeError(f"cannot stat {p}: {exc}") from exc - if size > _MERGE_MAX_BYTES: - raise RuntimeError( - f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" - ) - data = json.loads(path_obj.read_text(encoding="utf-8")) - try: - return _jg.node_link_graph(data, edges="links"), data - except TypeError: - return _jg.node_link_graph(data), data - try: - G_cur, _ = _load_graph(_current_path) - G_oth, _ = _load_graph(_other_path) - except Exception as exc: - print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) - sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge - merged = _nx.compose(G_cur, G_oth) - if merged.number_of_nodes() > _MERGE_MAX_NODES: - print( - f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " - f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", - file=sys.stderr, - ) - sys.exit(1) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - from graphify.paths import write_json_atomic - write_json_atomic(_current_path, out_data, indent=2) - sys.exit(0) - - elif cmd == "merge-graphs": - # graphify merge-graphs graph1.json graph2.json ... --out merged.json - args = sys.argv[2:] - graph_paths: list[Path] = [] - out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" - i = 0 - while i < len(args): - if args[i] == "--out" and i + 1 < len(args): - out_path = Path(args[i + 1]) - i += 2 - else: - graph_paths.append(Path(args[i])) - i += 1 - if len(graph_paths) < 2: - print( - "Usage: graphify merge-graphs [...] [--out merged.json]", - file=sys.stderr, - ) - sys.exit(1) - import networkx as _nx - from networkx.readwrite import json_graph as _jg - from graphify.build import prefix_graph_for_global as _prefix, distinct_repo_tags as _repo_tags - graphs = [] - for gp in graph_paths: - if not gp.exists(): - print(f"error: not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - data = json.loads(gp.read_text(encoding="utf-8")) - # Normalize edges/links key before loading — graphify writes "links" - # via node_link_data but older runs may have used "edges" (#738). - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - G = _jg.node_link_graph(data, edges="links") - except TypeError: - G = _jg.node_link_graph(data) - graphs.append(G) - # nx.compose requires all graphs to be the same type. When input graphs - # come from different sources (e.g. an AST-only run vs a full LLM run) one - # may be a MultiGraph and another a Graph. Normalise everything to Graph - # (the graphify default) by converting MultiGraphs with nx.Graph(). - def _to_simple(g: "_nx.Graph") -> "_nx.Graph": - # nx.compose requires every graph to be the same type. Inputs may - # disagree on BOTH axes — directed vs undirected, and multi vs simple - # — because per-repo graph.json files are written by different extract - # paths at different times. Normalise everything to a plain undirected - # Graph (the merged cross-repo view is undirected anyway), which covers - # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input - # crashed compose with "All graphs must be directed or undirected" (#1606). - if type(g) is not _nx.Graph: - return _nx.Graph(g) - return g - # Unique repo tag per graph. The bare `graphify-out/..` dir name is not - # unique across inputs (src/graphify-out and frontend/src/graphify-out both - # → "src"), which collides same-stem node ids and silently merges unrelated - # entities (#1729). distinct_repo_tags guarantees a distinct prefix per graph. - repo_tags = _repo_tags(graph_paths) - naive_tags = [gp.parent.parent.name for gp in graph_paths] - if len(set(naive_tags)) != len(naive_tags): - print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") - merged = _nx.Graph() - for G, repo_tag in zip(graphs, repo_tags): - prefixed = _to_simple(_prefix(G, repo_tag)) - merged = _nx.compose(merged, prefixed) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - out_path.parent.mkdir(parents=True, exist_ok=True) - from graphify.paths import write_json_atomic as _wja - _wja(out_path, out_data, indent=2) - print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") - print(f"Written to: {out_path}") - - elif cmd == "clone": - if len(sys.argv) < 3: - print( - "Usage: graphify clone [--branch ] [--out ]", - file=sys.stderr, - ) - sys.exit(1) - url = sys.argv[2] - branch: str | None = None - out_dir: Path | None = None - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--branch" and i + 1 < len(args): - branch = args[i + 1] - i += 2 - elif args[i] == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - local_path = _clone_repo(url, branch=branch, out_dir=out_dir) - print(local_path) - - elif cmd == "export": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd not in ("html", "callflow-html", "obsidian", "wiki", "svg", "graphml", "neo4j", "falkordb"): - print("Usage: graphify export ", file=sys.stderr) - print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) - print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) - print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) - print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) - print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" graphml [--graph PATH]", file=sys.stderr) - print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - sys.exit(1) - - # Parse shared args - args = sys.argv[3:] - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - graph_path_explicit = False - labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" - labels_path_explicit = False - report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" - report_path_explicit = False - sections_path: Path | None = None - callflow_output: Path | None = None - callflow_lang = "auto" - callflow_max_sections = 15 - callflow_diagram_scale = 1.0 - callflow_max_diagram_nodes = 18 - callflow_max_diagram_edges = 24 - analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" - node_limit = 5000 - no_viz = False - obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" - # Shared push-connection settings for the graph-database sinks (neo4j, - # falkordb), parsed from the generic --push/--user/--password flags below. - push_uri: str | None = None - push_user = "neo4j" # Neo4j default user; FalkorDB auth is optional and ignores it - # F-031: prefer an env var so the password never appears on argv (visible - # in `ps` output / shell history). The explicit --password flag still - # overrides it. Each sink reads its own var: FALKORDB_PASSWORD for falkordb, - # NEO4J_PASSWORD otherwise. - push_password: str | None = ( - os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" - else os.environ.get("NEO4J_PASSWORD") - ) or None - i = 0 - while i < len(args): - a = args[i] - if a == "--graph" and i + 1 < len(args): - graph_path = Path(args[i + 1]) - graph_path_explicit = True - i += 2 - elif a == "--labels" and i + 1 < len(args): - labels_path = Path(args[i + 1]) - labels_path_explicit = True - i += 2 - elif a == "--report" and i + 1 < len(args): - report_path = Path(args[i + 1]) - report_path_explicit = True - i += 2 - elif a == "--sections" and i + 1 < len(args): - sections_path = Path(args[i + 1]); i += 2 - elif a == "--output" and i + 1 < len(args): - callflow_output = Path(args[i + 1]).expanduser() - if not callflow_output.is_absolute(): - callflow_output = Path.cwd() / callflow_output - i += 2 - elif a == "--lang" and i + 1 < len(args): - callflow_lang = args[i + 1]; i += 2 - elif a == "--max-sections" and i + 1 < len(args): - callflow_max_sections = int(args[i + 1]); i += 2 - elif a == "--diagram-scale" and i + 1 < len(args): - callflow_diagram_scale = float(args[i + 1]); i += 2 - elif a == "--max-diagram-nodes" and i + 1 < len(args): - callflow_max_diagram_nodes = int(args[i + 1]); i += 2 - elif a == "--max-diagram-edges" and i + 1 < len(args): - callflow_max_diagram_edges = int(args[i + 1]); i += 2 - elif a in ("-h", "--help") and subcmd == "callflow-html": - print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") - print(" --report PATH path to GRAPH_REPORT.md") - print(" --sections PATH JSON section definitions") - print(" --output HTML output path (default graphify-out/-callflow.html)") - print(" --lang LANG auto, zh-CN, en, etc. (default auto)") - print(" --max-sections N maximum auto-derived sections (default 15)") - print(" --diagram-scale N Mermaid diagram scale (default 1.0)") - print(" --max-diagram-nodes N representative nodes per section (default 18)") - print(" --max-diagram-edges N representative edges per section (default 24)") - sys.exit(0) - elif a == "--node-limit" and i + 1 < len(args): - node_limit = int(args[i + 1]); i += 2 - elif a == "--no-viz": - no_viz = True; i += 1 - elif a == "--dir" and i + 1 < len(args): - obsidian_dir = Path(args[i + 1]); i += 2 - elif a == "--push" and i + 1 < len(args): - push_uri = args[i + 1]; i += 2 - elif a == "--user" and i + 1 < len(args): - push_user = args[i + 1]; i += 2 - elif a == "--password" and i + 1 < len(args): - push_password = args[i + 1]; i += 2 - elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: - candidate = Path(a) - if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": - graph_path = candidate - elif (candidate / "graph.json").exists(): - graph_path = candidate / "graph.json" - else: - graph_path = candidate / _GRAPHIFY_OUT / "graph.json" - graph_path_explicit = True - i += 1 - else: - i += 1 - - graph_path = graph_path.expanduser() - if graph_path_explicit: - graph_out_dir = graph_path.parent - if not labels_path_explicit: - labels_path = graph_out_dir / ".graphify_labels.json" - if not report_path_explicit: - report_path = graph_out_dir / "GRAPH_REPORT.md" - labels_path = labels_path.expanduser() - report_path = report_path.expanduser() - - if not graph_path.exists(): - print(f"error: graph not found: {graph_path}. Run /graphify first.", file=sys.stderr) - sys.exit(1) - - if subcmd == "callflow-html": - from graphify.callflow_html import write_callflow_html as _write_callflow_html - out = _write_callflow_html( - graph=graph_path, - report=report_path, - labels=labels_path, - sections=sections_path, - output=callflow_output, - lang=callflow_lang, - max_sections=callflow_max_sections, - diagram_scale=callflow_diagram_scale, - max_diagram_nodes=callflow_max_diagram_nodes, - max_diagram_edges=callflow_max_diagram_edges, - verbose=True, - ) - print(f"callflow HTML written - open in any browser: {out}") - sys.exit(0) - - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json as _bfj - from graphify.security import check_graph_file_size_cap as _check_cap - - # Solution 3 (#1019): for the HTML view, an oversized graph.json should - # not be a hard error. Detect the over-cap condition here and fall back - # to the community-aggregation view (node_limit=5000) below instead of - # exiting 1. All other subcommands keep the hard cap. - _over_cap = False - try: - _check_cap(graph_path) - except ValueError as _cap_err: - if subcmd == "html": - _over_cap = True - try: - _over_cap_bytes = graph_path.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - else: - print(f"error: {_cap_err}", file=sys.stderr) - sys.exit(1) - _raw = json.loads(graph_path.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = _jg.node_link_graph(_raw, edges="links") - except TypeError: - G = _jg.node_link_graph(_raw) - - # Load optional analysis/labels - communities: dict[int, list[str]] = {} - if analysis_path.exists(): - _an = json.loads(analysis_path.read_text(encoding="utf-8")) - communities = {int(k): v for k, v in _an.get("communities", {}).items()} - cohesion: dict[int, float] = {int(k): v for k, v in _an.get("cohesion", {}).items()} - gods_data = _an.get("gods", []) - else: - cohesion = {} - gods_data = [] - - # Fallback: graph.json carries the per-node community as a node attribute - # (`to_json` writes it on every node). The analysis sidecar is the - # canonical source — but the post-commit / watch rebuild path doesn't - # regenerate it, and `extract` may have its temp files cleaned up. When - # that happens, `graphify export html` previously bailed with - # "Single community - aggregated view not useful." even though the - # per-node attribute had the right data all along. Reconstruct from - # the graph itself so downstream subcommands (html, obsidian, wiki, - # svg, graphml, neo4j) don't silently produce a degraded artifact. - if not communities: - reconstructed: dict[int, list[str]] = {} - for node_id, data in G.nodes(data=True): - cid_raw = data.get("community") - if cid_raw is None: - continue - try: - cid = int(cid_raw) - except (TypeError, ValueError): - continue - reconstructed.setdefault(cid, []).append(str(node_id)) - if reconstructed: - communities = reconstructed - - labels: dict[int, str] = {} - if labels_path.exists(): - labels = {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} - - out_dir = graph_path.parent - - if subcmd == "html": - from graphify.export import to_html as _to_html - if no_viz: - html_target = out_dir / "graph.html" - if html_target.exists(): - html_target.unlink() - print("--no-viz: skipped graph.html") - else: - # Over-cap fallback (#1019): force the community-aggregation - # path so the oversized graph still renders a usable artifact. - _effective_node_limit = 5000 if _over_cap else node_limit - _to_html(G, communities, str(out_dir / "graph.html"), - community_labels=labels or None, node_limit=_effective_node_limit) - if G.number_of_nodes() <= _effective_node_limit: - print(f"graph.html written - open in any browser, no server needed") - if _over_cap: - sys.exit(0) - - elif subcmd == "obsidian": - from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas - n = _to_obsidian(G, communities, str(obsidian_dir), - community_labels=labels or None, cohesion=cohesion or None) - print(f"Obsidian vault: {n} notes in {obsidian_dir}/") - _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), - community_labels=labels or None) - print(f"Canvas: {obsidian_dir}/graph.canvas") - print(f"Open {obsidian_dir}/ as a vault in Obsidian.") - - elif subcmd == "wiki": - from graphify.wiki import to_wiki as _to_wiki - from graphify.analyze import god_nodes as _god_nodes - if not communities: - print( - "error: .graphify_analysis.json is missing or empty — refusing to export wiki to prevent data loss.\n" - "Run `graphify extract .` (or `graphify cluster-only .`) to regenerate community data first.", - file=sys.stderr, - ) - sys.exit(1) - if not gods_data: - gods_data = _god_nodes(G) - n = _to_wiki(G, communities, str(out_dir / "wiki"), - community_labels=labels or None, cohesion=cohesion or None, - god_nodes_data=gods_data) - print(f"Wiki: {n} articles written to {out_dir}/wiki/") - print(f" {out_dir}/wiki/index.md -> agent entry point") - - elif subcmd == "svg": - from graphify.export import to_svg as _to_svg - _to_svg(G, communities, str(out_dir / "graph.svg"), - community_labels=labels or None) - print(f"graph.svg written - embeds in Obsidian, Notion, GitHub READMEs") - - elif subcmd == "graphml": - from graphify.export import to_graphml as _to_graphml - _to_graphml(G, communities, str(out_dir / "graph.graphml")) - print(f"graph.graphml written - open in Gephi, yEd, or any GraphML tool") - - elif subcmd == "neo4j": - if push_uri: - from graphify.export import push_to_neo4j as _push - if push_password is None: - print("error: --password required for --push", file=sys.stderr) - sys.exit(1) - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written - import with: cypher-shell < {out_dir}/cypher.txt") - - elif subcmd == "falkordb": - if push_uri: - from graphify.export import push_to_falkordb as _push - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written ({out_dir}/cypher.txt) - statements are OpenCypher. " - f"FalkorDB's GRAPH.QUERY runs one statement at a time (no bulk script " - f"import), so load a graph with: graphify export falkordb --push " - f"falkordb://localhost:6379") - - elif cmd == "benchmark": - from graphify.benchmark import run_benchmark, print_benchmark - - graph_path = sys.argv[2] if len(sys.argv) > 2 else _default_graph_path() - _enforce_graph_size_cap_or_exit(Path(graph_path)) - # Try to load corpus_words from detect output - corpus_words = None - detect_path = Path(".graphify_detect.json") - if detect_path.exists(): - try: - detect_data = json.loads(detect_path.read_text(encoding="utf-8")) - corpus_words = detect_data.get("total_words") - except Exception: - pass - result = run_benchmark(graph_path, corpus_words=corpus_words) - print_benchmark(result) - - elif cmd == "global": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - from graphify.global_graph import ( - global_add as _global_add, - global_remove as _global_remove, - global_list as _global_list, - global_path as _global_path, + if not _rebuild_code( + target, output_root=output, changed_paths=changed, force=force, no_cluster=no_cluster, + block_on_lock=True, + include_semantic=include_semantic, + code_only="--code-only" in args, + backend=_option(args, "--backend"), + model=_option(args, "--model"), + deep_mode=_option(args, "--mode") == "deep" or "--deep" in args, + token_budget=int(_option(args, "--token-budget") or 60_000), + max_concurrency=int(_option(args, "--max-concurrency") or 4), + retain_rollback="--retain-rollback" in args, + raise_on_error=True, + ): + raise RuntimeError("graph rebuild failed") + + +def _save_result(args: list[str]) -> None: + from graphify.ingest import save_query_result + + question = _option(args, "--question") + answer = _option(args, "--answer") + answer_file = _option(args, "--answer-file") + if answer_file: + answer = Path(answer_file).read_text(encoding="utf-8").strip() + if not question or not answer: + raise ValueError("save-result requires --question and --answer or --answer-file") + nodes: list[str] = [] + if "--nodes" in args: + index = args.index("--nodes") + 1 + while index < len(args) and not args[index].startswith("--"): + nodes.append(args[index]) + index += 1 + output = save_query_result( + question=question, + answer=answer, + memory_dir=Path(_option(args, "--memory-dir") or Path(_GRAPHIFY_OUT) / "memory"), + query_type=_option(args, "--type") or "query", + source_nodes=nodes or None, + outcome=_option(args, "--outcome"), + correction=_option(args, "--correction"), + ) + print(f"Saved to {output}") + + +def _reflect(args: list[str]) -> None: + from graphify.reflect import lessons_fresh, reflect + + memory_dir = Path(_option(args, "--memory-dir") or Path(_GRAPHIFY_OUT) / "memory") + output = Path(_option(args, "--out") or Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md") + store_value = _option(args, "--store", "--graph") + store = _store_arg(args) if store_value else None + if "--if-stale" in args and lessons_fresh(output, memory_dir, store): + print(f"Lessons already up to date -> {output}") + return + result, aggregate = reflect( + memory_dir=memory_dir, + out_path=output, + graph_path=store, + half_life_days=float(_option(args, "--half-life-days") or 30), + min_corroboration=int(_option(args, "--min-corroboration") or 2), + ) + print(f"Reflected {aggregate['total']} memories -> {result}") + + +def _tree(args: list[str]) -> None: + from graphify.tree_html import write_tree_html + + store = _store_arg(args) + output = Path(_option(args, "--output", "--out") or Path(_GRAPHIFY_OUT) / "GRAPH_TREE.html") + result = write_tree_html( + store, + output, + root=_option(args, "--root"), + max_children=int(_option(args, "--max-children") or 200), + project_label=_option(args, "--label"), + top_k_edges=int(_option(args, "--top-k-edges") or 12), + ) + print(result) + + +def _cache_check(args: list[str]) -> None: + from graphify.cache import check_semantic_cache + from graphify.helix.persistence import load_graph + + if not args or args[0].startswith("-"): + raise ValueError("cache-check requires a newline-delimited file list") + root = Path(_option(args, "--root") or ".") + files = [line for line in Path(args[0]).read_text(encoding="utf-8").splitlines() if line.strip()] + store = root / _GRAPHIFY_OUT / "graph.helix" + cache: dict[str, dict] = {} + if store.is_dir(): + loaded = load_graph(store) + value = loaded.state.get("incremental", {}).get("extraction_cache", {}) + if isinstance(value, dict): + cache = value + nodes, edges, hyperedges, uncached = check_semantic_cache( + files, + cache, + root=root, + mode="deep" if "--deep" in args else _option(args, "--mode"), + prompt_file=_option(args, "--prompt-file"), + ) + # These are transient extraction interchange files, never graph stores. + out = root / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + if nodes or edges or hyperedges: + (out / ".graphify_cached.json").write_text( + json.dumps({"nodes": nodes, "edges": edges, "hyperedges": hyperedges}), + encoding="utf-8", ) - if subcmd == "add": - # graphify global add [--as ] - args = sys.argv[3:] - source = None - tag = None - i = 0 - while i < len(args): - if args[i] == "--as" and i + 1 < len(args): - tag = args[i + 1]; i += 2 - elif not source: - source = Path(args[i]); i += 1 - else: - i += 1 - if not source: - print("Usage: graphify global add [--as ]", file=sys.stderr) - sys.exit(1) - tag = tag or source.parent.parent.name - try: - result = _global_add(source, tag) - if result["skipped"]: - print(f"'{tag}' unchanged since last add - global graph not modified.") - else: - print(f"Added '{tag}' to global graph: +{result['nodes_added']} nodes, " - f"-{result['nodes_removed']} pruned. Global: {_global_path()}") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "remove": - tag = sys.argv[3] if len(sys.argv) > 3 else "" - if not tag: - print("Usage: graphify global remove ", file=sys.stderr); sys.exit(1) - try: - removed = _global_remove(tag) - print(f"Removed '{tag}' from global graph ({removed} nodes pruned).") - except KeyError as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "list": - repos = _global_list() - if not repos: - print("Global graph is empty. Use 'graphify global add' to add a project.") - else: - print(f"Global graph: {_global_path()}") - for tag, info in repos.items(): - print(f" {tag}: {info.get('node_count', '?')} nodes, added {info.get('added_at', '?')[:10]}") - elif subcmd == "path": - print(_global_path()) - else: - print("Usage: graphify global [add|remove|list|path]", file=sys.stderr); sys.exit(1) - - elif cmd == "extract": - # Headless full-pipeline extraction for CI / scripts (#698). - # Runs detect -> AST extraction on code -> semantic LLM extraction on - # docs/papers/images -> merge -> build -> cluster -> write outputs. - # Unlike the skill.md path (which runs through Claude Code subagents), - # this calls extract_corpus_parallel directly using whichever backend - # has an API key set. - if len(sys.argv) < 3: - print( - "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " - "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " - "[--no-gitignore] " - "[--max-workers N] [--token-budget N] [--max-concurrency N] " - "[--api-timeout S] [--postgres DSN] [--cargo] [--allow-partial] [--timing]", - file=sys.stderr, - ) - sys.exit(1) + (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") + print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") - has_path = True - if sys.argv[2].startswith("-"): - has_path = False - target = Path(".").resolve() - else: - target = Path(sys.argv[2]).resolve() - if not target.exists(): - print(f"error: path not found: {target}", file=sys.stderr) - sys.exit(1) - - backend: str | None = None - model: str | None = None - extract_mode: str | None = None - out_dir: Path | None = None - cli_postgres_dsn: str | None = None - cli_cargo: bool = False - cli_allow_partial: bool = False - no_cluster = False - dedup_llm = False - google_workspace = False - global_merge = False - code_only = False - no_gitignore = False - global_repo_tag: str | None = None - # Performance/tuning knobs (issue #792). None means "use library default". - cli_max_workers: int | None = None - cli_token_budget: int | None = None - cli_max_concurrency: int | None = None - cli_api_timeout: float | None = None - # Clustering tuning knobs - cli_resolution: float = 1.0 - cli_exclude_hubs: float | None = None - cli_excludes: list[str] = [] - cli_timing: bool = False - # --force parity with `graphify update`: the flag or GRAPHIFY_FORCE=1 - # disables the incremental gate and skips semantic-cache reads (#1894). - force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") - - def _parse_int(name: str, raw: str) -> int: - try: - v = int(raw) - except ValueError: - print(f"error: {name} must be a positive integer (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - def _parse_float(name: str, raw: str) -> float: - try: - v = float(raw) - except ValueError: - print(f"error: {name} must be a positive number (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - args = sys.argv[3:] if has_path else sys.argv[2:] - i = 0 - while i < len(args): - a = args[i] - if a == "--backend" and i + 1 < len(args): - backend = args[i + 1]; i += 2 - elif a.startswith("--backend="): - backend = a.split("=", 1)[1]; i += 1 - elif a == "--model" and i + 1 < len(args): - model = args[i + 1]; i += 2 - elif a.startswith("--model="): - model = a.split("=", 1)[1]; i += 1 - elif a == "--mode" and i + 1 < len(args): - extract_mode = args[i + 1]; i += 2 - elif a.startswith("--mode="): - extract_mode = a.split("=", 1)[1]; i += 1 - elif a == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]); i += 2 - elif a.startswith("--out="): - out_dir = Path(a.split("=", 1)[1]); i += 1 - elif a == "--no-cluster": - no_cluster = True; i += 1 - elif a == "--dedup-llm": - dedup_llm = True; i += 1 - elif a == "--code-only": - code_only = True; i += 1 - elif a == "--google-workspace": - google_workspace = True; i += 1 - elif a == "--no-gitignore": - no_gitignore = True; i += 1 - elif a == "--global": - global_merge = True; i += 1 - elif a == "--as" and i + 1 < len(args): - global_repo_tag = args[i + 1]; i += 2 - elif a == "--max-workers" and i + 1 < len(args): - cli_max_workers = _parse_int("--max-workers", args[i + 1]); i += 2 - elif a.startswith("--max-workers="): - cli_max_workers = _parse_int("--max-workers", a.split("=", 1)[1]); i += 1 - elif a == "--token-budget" and i + 1 < len(args): - cli_token_budget = _parse_int("--token-budget", args[i + 1]); i += 2 - elif a.startswith("--token-budget="): - cli_token_budget = _parse_int("--token-budget", a.split("=", 1)[1]); i += 1 - elif a == "--max-concurrency" and i + 1 < len(args): - cli_max_concurrency = _parse_int("--max-concurrency", args[i + 1]); i += 2 - elif a.startswith("--max-concurrency="): - cli_max_concurrency = _parse_int("--max-concurrency", a.split("=", 1)[1]); i += 1 - elif a == "--api-timeout" and i + 1 < len(args): - cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 - elif a.startswith("--api-timeout="): - cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 - elif a == "--resolution" and i + 1 < len(args): - cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 - elif a.startswith("--resolution="): - cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 - elif a == "--exclude-hubs" and i + 1 < len(args): - cli_exclude_hubs = float(args[i + 1]); i += 2 - elif a.startswith("--exclude-hubs="): - cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 - elif a == "--exclude" and i + 1 < len(args): - cli_excludes.append(args[i + 1]); i += 2 - elif a.startswith("--exclude="): - cli_excludes.append(a.split("=", 1)[1]); i += 1 - elif a == "--postgres" and i + 1 < len(args): - cli_postgres_dsn = args[i + 1]; i += 2 - elif a.startswith("--postgres="): - cli_postgres_dsn = a.split("=", 1)[1]; i += 1 - elif a == "--cargo": - cli_cargo = True - i += 1 - elif a == "--force": - force = True; i += 1 - elif a == "--allow-partial": - cli_allow_partial = True; i += 1 - elif a == "--timing": - cli_timing = True; i += 1 - else: - i += 1 - - if not has_path and cli_postgres_dsn is None: - print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) - sys.exit(1) - - _VALID_MODES = {"deep"} - if extract_mode is not None and extract_mode not in _VALID_MODES: - print( - f"error: unknown --mode '{extract_mode}'. " - f"Available: {', '.join(sorted(_VALID_MODES))}", - file=sys.stderr, - ) - sys.exit(2) - deep_mode = extract_mode == "deep" - if deep_mode: - print("[graphify extract] deep mode enabled: richer semantic extraction") - - # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so - # _call_openai_compat picks it up without needing a new kwarg path. - if cli_api_timeout is not None: - os.environ["GRAPHIFY_API_TIMEOUT"] = str(cli_api_timeout) - if cli_max_workers is not None: - os.environ["GRAPHIFY_MAX_WORKERS"] = str(cli_max_workers) - - # Resolve output dir. The user-facing contract is "/graphify-out/" - # so a fresh checkout writes graphify-out/ at the project root, matching - # the skill.md pipeline. - out_root = (out_dir.resolve() if out_dir else target) - graphify_out = out_root / _GRAPHIFY_OUT - graphify_out.mkdir(parents=True, exist_ok=True) - # Persist corpus-shaping options so later update/watch/hook rebuilds - # use the same file set as the initial extraction (#1886). - from graphify.watch import ( - _write_build_config as _write_build_cfg, - _read_build_gitignore as _read_build_gi, - ) - # #1971 persistence: an explicit --no-gitignore persists False; a later - # flag-less `graphify extract` must NOT clobber it back to True, which - # would make the git-ignored code silently disappear again (the exact - # complaint #1971 is about). Honor the persisted value for THIS run when - # the flag is absent (read before the write below), and write False only - # when the flag is set — None leaves the setting as-is, mirroring how - # #1886 persists --exclude. - _effective_gitignore = False if no_gitignore else _read_build_gi(graphify_out) - _write_build_cfg( - graphify_out, - excludes=cli_excludes or None, - gitignore=False if no_gitignore else None, - ) - stages = _StageTimer(cli_timing) +def _rollback(args: list[str]) -> None: + from graphify.helix.persistence import HelixEmbeddedStore + from graphify.security import validate_store_path - from graphify.detect import ( - detect as _detect, - detect_incremental as _detect_incremental, - save_manifest as _save_manifest, - ) - manifest_path = graphify_out / "manifest.json" - existing_graph_path = graphify_out / "graph.json" - # #1925: a missing manifest.json must not degrade to a full scan that - # discards the existing graph's semantic layer. An existing graph.json - # is a sufficient incremental baseline: detect_incremental treats an - # absent manifest as "everything is new" (re-extract all, nothing - # deleted), and build_merge + _stale_graph_sources reconcile replaced - # and genuinely-deleted sources against the current corpus, so doc/ - # paper/image nodes survive a --code-only rebuild instead of being - # dropped with the rest of the committed graph. - incremental_mode = existing_graph_path.exists() if has_path else False - # --force: full scan, not the manifest-gated incremental diff — a warm - # unchanged tree would otherwise dispatch zero files (#1894). - incremental_mode = incremental_mode and not force - if force: - print("[graphify extract] --force: full re-scan, semantic cache reads skipped") - elif incremental_mode and not manifest_path.exists(): - print( - "[graphify extract] manifest.json missing; using existing " - "graph.json as the incremental baseline (all files re-checked; " - "nodes for files outside this run's scope are preserved)" - ) + store_path = validate_store_path(_store_arg(args)) + with HelixEmbeddedStore(store_path, retain_rollback=True) as store: + loaded = store.rollback() + print(f"Rolled back {store_path} to generation {loaded.generation}.") - if not has_path: - code_files = [] - doc_files = [] - paper_files = [] - image_files = [] - deleted_files = [] - excluded_files = [] - graph_stale_sources = [] - unchanged_total = 0 - files_by_type = {} - elif incremental_mode: - print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental( - target, - manifest_path=str(manifest_path), - google_workspace=google_workspace or None, - extra_excludes=cli_excludes or None, - gitignore=_effective_gitignore, - ) - files_by_type = detection.get("files", {}) - new_by_type = detection.get("new_files", {}) - code_files = [Path(p) for p in new_by_type.get("code", [])] - doc_files = [Path(p) for p in new_by_type.get("document", [])] - paper_files = [Path(p) for p in new_by_type.get("paper", [])] - image_files = [Path(p) for p in new_by_type.get("image", [])] - deleted_files = list(detection.get("deleted_files", [])) - excluded_files = list(detection.get("excluded_files", [])) - unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) - # #1909: derive the prune set from the existing graph itself, not - # just the manifest. A file that became excluded without ever - # being manifest-listed (every pre-#1897 graph is in this state) - # still has stale nodes carried forward by build_merge unless the - # graph's own sources are reconciled against the current corpus. - _seen_files = {f for _fl in files_by_type.values() for f in _fl} - _seen_files.update(detection.get("unclassified", [])) - graph_stale_sources = _stale_graph_sources( - existing_graph_path, target, _seen_files - ) - else: - print(f"[graphify extract] scanning {target}") - detection = _detect( - target, - google_workspace=google_workspace or None, - extra_excludes=cli_excludes or None, - cache_root=out_root, - gitignore=_effective_gitignore, - ) - files_by_type = detection.get("files", {}) - code_files = [Path(p) for p in files_by_type.get("code", [])] - doc_files = [Path(p) for p in files_by_type.get("document", [])] - paper_files = [Path(p) for p in files_by_type.get("paper", [])] - image_files = [Path(p) for p in files_by_type.get("image", [])] - deleted_files = [] - excluded_files = [] - graph_stale_sources = [] - unchanged_total = 0 - - semantic_files = doc_files + paper_files + image_files - # --code-only: index code (pure local AST, no key) and skip the semantic - # (doc/paper/image) pass entirely, so a mixed repo doesn't hard-fail when no - # LLM backend is configured (#1734). Report what was skipped rather than - # silently dropping it. - if code_only and semantic_files: - print( - f"[graphify extract] --code-only: skipping {len(semantic_files)} " - f"non-code file(s) ({len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images) — no LLM extraction" - ) - semantic_files = [] - doc_files = [] - paper_files = [] - image_files = [] - if deep_mode and incremental_mode and not code_only: - # Deep mode reads/writes its own cache namespace - # (cache/semantic-deep/), so the manifest's changed-file gate is - # not a valid proxy for deep coverage: over a warm unchanged tree - # it dispatches zero files and `--mode deep` silently no-ops - # (#1894). Widen the semantic pass to the FULL live - # doc/paper/image set (``files_by_type`` from detect_incremental, - # which already excludes excluded files) and let the - # mode-namespaced cache decide hits/misses — the first deep run - # re-dispatches everything (deep namespace cold), later deep runs - # hit the deep cache. - _deep_all = [ - Path(p) - for _ftype in ("document", "paper", "image") - for p in files_by_type.get(_ftype, []) - ] - if len(_deep_all) != len(semantic_files): - print( - f"[graphify extract] deep mode: widening semantic pass from " - f"{len(semantic_files)} changed to {len(_deep_all)} live " - f"doc/paper/image file(s); the deep semantic cache decides " - f"what is re-extracted" - ) - semantic_files = _deep_all - if incremental_mode: - # Excluded-but-alive files are reported separately from deletions - # (#1908): they still exist on disk, the scan just stopped - # covering them (ignore rules / --exclude changed). - _excl_note = f"; {len(excluded_files)} excluded" if excluded_files else "" - print( - f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " - f"{len(paper_files)} papers, {len(image_files)} images changed; " - f"{unchanged_total} unchanged; {len(deleted_files)} deleted" - f"{_excl_note}" - ) - else: - print( - f"[graphify extract] found {len(code_files)} code, " - f"{len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images" - ) - # Surface files that were seen but not classified (extensionless non-shebang - # project files like Dockerfile/Makefile, or unsupported extensions), so they - # are no longer invisible in graphify's own output (#1692). - _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] - if _unclassified: - _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) - _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" - print( - f"[graphify extract] {len(_unclassified)} file(s) not classified " - f"(no supported extension or shebang), skipped: {_names}{_more}" - ) - stages.mark("detect") - - # Resolve the LLM backend only now that we know whether the corpus - # needs one. A code-only corpus is pure local AST and must not require - # an API key; the key is enforced below only when there's LLM work. - from graphify.llm import ( - BACKENDS as _BACKENDS, - detect_backend as _detect_backend, - estimate_cost as _estimate_cost, - extract_corpus_parallel as _extract_corpus_parallel, - _format_backend_env_keys, - _get_backend_api_key, - ) - needs_llm = bool(semantic_files) or dedup_llm - if backend is None and needs_llm: - backend = _detect_backend() - if backend is not None and backend not in _BACKENDS: - print( - f"error: unknown backend '{backend}'. " - f"Available: {', '.join(sorted(_BACKENDS))}", - file=sys.stderr, - ) - sys.exit(1) - if needs_llm: - if backend is None: - reasons = [] - if semantic_files: - reasons.append( - f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" - ) - if dedup_llm: - reasons.append("--dedup-llm was passed") - hint = "" - if semantic_files: - hint = (" Or pass --code-only to index just the code " - "(local AST, no key) and skip the non-code files.") - print( - "error: no LLM API key found (" + "; ".join(reasons) + "). " - "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " - "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " - "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " - "corpus needs no key." + hint, - file=sys.stderr, - ) - sys.exit(1) - if backend == "ollama": - from graphify.llm import _validate_ollama_base_url - _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) - try: - _validate_ollama_base_url(_oll_url, warn=False) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(2) - if not _get_backend_api_key(backend): - allow_no_key = False - if backend == "ollama": - from urllib.parse import urlparse - ollama_url = os.environ.get( - "OLLAMA_BASE_URL", - _BACKENDS["ollama"].get("base_url", ""), - ) - try: - host = (urlparse(ollama_url).hostname or "").lower() - except Exception: - host = "" - allow_no_key = ( - host in ("localhost", "127.0.0.1", "::1") - or host.startswith("127.") - ) - elif backend == "bedrock": - allow_no_key = bool( - os.environ.get("AWS_PROFILE") - or os.environ.get("AWS_REGION") - or os.environ.get("AWS_DEFAULT_REGION") - or os.environ.get("AWS_ACCESS_KEY_ID") - ) - elif backend == "claude-cli": - import shutil as _shutil - allow_no_key = _shutil.which("claude") is not None - if not allow_no_key: - print( - "error: backend 'claude-cli' requires the `claude` CLI on $PATH " - "(install Claude Code and run `claude` once to authenticate).", - file=sys.stderr, - ) - sys.exit(1) - if not allow_no_key: - print( - f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", - file=sys.stderr, - ) - sys.exit(1) - - # Track whether this run's extraction was incomplete (a whole extractor - # pass crashed, or some semantic chunks failed). A partial result must not - # be force-written over a good complete graph — the final write falls back - # to the #479 shrink guard unless --allow-partial is set. - _extraction_incomplete = False - # A walk that couldn't fully enumerate the corpus (permission-denied - # subtree, I/O error) yields a legitimately smaller graph that must not - # be force-written over a complete one — same failure class as a crashed - # pass. detect()/detect_incremental() already record these; consume them. - if detection.get("walk_errors"): - _extraction_incomplete = True - - # AST extraction on code files. Empty code list (docs-only corpus) is - # the issue #698 case — skip cleanly instead of crashing inside extract(). - ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - if code_files: - from graphify.extract import extract as _ast_extract - # Anchor the cache at the output root, not the scanned project: - # with --out, a /graphify-out/cache/ would leak a - # graphify-out/ dir into a project that asked for external output. - # `root` stays the scanned project so source_file/ids relativize - # against it; conflating the two basenamed every node (#1941). - ast_kwargs: dict = {"cache_root": out_root, "root": target} - if cli_max_workers is not None: - ast_kwargs["max_workers"] = cli_max_workers - print(f"[graphify extract] AST extraction on {len(code_files)} code files...") - try: - ast_result = _ast_extract(code_files, **ast_kwargs) - except Exception as exc: - print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) - ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - _extraction_incomplete = True # the whole AST pass was lost - stages.mark("AST extract") - - # Semantic extraction on docs/papers/images. Check cache first. - from graphify.cache import ( - check_semantic_cache as _check_semantic_cache, - prune_semantic_cache as _prune_semantic_cache, - save_semantic_cache as _save_semantic_cache, - ) - sem_result: dict = { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, - } - # Semantic files whose extraction truncated this run. They are left - # unstamped in the manifest so detect_incremental re-queues them next run - # (mirrors the #933 failed-chunk handling); captured below before the - # _partial markers are stripped from the corpus. - _partial_semantic_files: set[str] = set() - sem_cache_hits = 0 - sem_cache_misses = 0 - # Deep mode uses its own namespace (cache/semantic-deep/) so deep and - # standard results for the same content never shadow each other (#1894). - sem_cache_mode = "deep" if deep_mode else None - # Entries are attributed to the extraction prompt that produced them, so - # a release that changes the prompt re-extracts rather than replaying the - # older vintage alongside the new one (#1939). Read and write must pass - # the same prompt, or the write lands where the next read won't look. - from graphify.llm import _extraction_system as _sem_prompt_for - sem_prompt = _sem_prompt_for(deep=deep_mode) - if semantic_files: - sem_paths_str = [str(p) for p in semantic_files] - if force: - # --force: skip the cache READ so every semantic file is - # re-dispatched; the save below still runs so the fresh - # results replace the stale entries. - cached_nodes, cached_edges, cached_hyperedges = [], [], [] - uncached_paths = list(sem_paths_str) - else: - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=target, cache_root=out_root, - mode=sem_cache_mode, prompt=sem_prompt) - ) - sem_cache_hits = len(semantic_files) - len(uncached_paths) - sem_cache_misses = len(uncached_paths) - sem_result["nodes"].extend(cached_nodes) - sem_result["edges"].extend(cached_edges) - sem_result["hyperedges"].extend(cached_hyperedges) - if sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") - - if uncached_paths: - print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") - corpus_kwargs: dict = { - "backend": backend, - "model": model, - "root": target, - "cache_root": out_root, - } - if deep_mode: - corpus_kwargs["deep_mode"] = True - if cli_token_budget is not None: - corpus_kwargs["token_budget"] = cli_token_budget - if cli_max_concurrency is not None: - corpus_kwargs["max_concurrency"] = cli_max_concurrency - - # Minimal progress callback so the CLI is no longer silent - # during long local-inference runs (issue #792 addendum). - # Also track per-chunk success so we can fail loudly when - # every chunk errors (e.g. missing backend SDK package). - _chunk_stats = {"total": 0, "succeeded": 0} - def _progress(idx: int, total: int, _result: dict) -> None: - _chunk_stats["total"] = total - _chunk_stats["succeeded"] += 1 - print( - f"[graphify extract] chunk {idx + 1}/{total} done", - flush=True, - ) - corpus_kwargs["on_chunk_done"] = _progress - try: - fresh = _extract_corpus_parallel( - [Path(p) for p in uncached_paths], - **corpus_kwargs, - ) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - except Exception as exc: - print( - f"[graphify extract] semantic extraction failed: {exc}", - file=sys.stderr, - ) - fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - _extraction_incomplete = True # the semantic pass crashed - - # on_chunk_done only fires after a chunk succeeds. If fresh - # semantic extraction was requested and no chunks completed, - # fail instead of writing an AST-only graph with exit 0. - if uncached_paths and _chunk_stats["succeeded"] == 0: - print( - f"[graphify extract] error: all semantic chunks failed " - f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " - f"see per-chunk errors above. If you see 'requires the X package', " - f"run `pip install X` and retry.", - file=sys.stderr, - ) - sys.exit(1) - # Some (but not all) chunks failed — the graph is missing nodes - # from the failed chunks, so it must not clobber a larger complete - # graph without an explicit --allow-partial override. - if _chunk_stats["total"] and _chunk_stats["succeeded"] < _chunk_stats["total"]: - _extraction_incomplete = True - # Which files truncated this run (item markers + the empty-parse - # _partial_files set). Computed BEFORE the save so it can be passed - # as partial_source_files: without it, a file whose only truncated - # chunk parsed empty (so it has no item markers here) would be - # written as a complete cache entry, re-promoting it (#1950). - from graphify.llm import ( - _partial_source_files as _partial_sf, - _strip_partial_markers as _strip_partial, - ) - _partial_semantic_files = set(_partial_sf(fresh)) - try: - _save_semantic_cache( - fresh.get("nodes", []), - fresh.get("edges", []), - fresh.get("hyperedges", []), - root=target, - cache_root=out_root, - allowed_source_files=uncached_paths, - mode=sem_cache_mode, - prompt=sem_prompt, - partial_source_files=_partial_semantic_files or None, - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - # Strip the markers before the corpus feeds the graph so the - # internal flag never leaks into graph.json. - _strip_partial(fresh) - sem_result["nodes"].extend(fresh.get("nodes", [])) - sem_result["edges"].extend(fresh.get("edges", [])) - sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) - sem_result["input_tokens"] += fresh.get("input_tokens", 0) - sem_result["output_tokens"] += fresh.get("output_tokens", 0) - - # Prune orphaned semantic cache entries. The semantic cache is - # content-hash-keyed and unversioned, so it is never swept by the AST - # version-cleanup: every content change or file deletion leaves a - # permanent orphan that accumulates unbounded (#1527). Sweep it against - # the FULL live document set (``files_by_type`` — present in both the - # incremental and full branches), NOT the incremental ``semantic_files`` - # changed-subset, which would delete every unchanged doc's valid entry. - # Best-effort: a prune failure must never break extraction. - # Hash keys are anchored to the corpus (``target``) — the same anchor - # the cache read/write above use — while the stat-index artifact - # follows the cache location (``out_root``). Anchoring these hashes to - # ``out_root`` instead would mismatch every key under ``--out`` and - # sweep the entire fresh cache as orphaned (#1990/#1991). - try: - from graphify.cache import file_hash as _file_hash - _live_hashes: set[str] = set() - for _kind in ("document", "paper", "image"): - for _fp in files_by_type.get(_kind, []): - _abs = Path(_fp) - if not _abs.is_absolute(): - _abs = Path(target) / _abs - if not _abs.is_file(): - continue # deleted/missing — leave out so its entry is pruned - try: - _live_hashes.add(_file_hash(_abs, target, cache_root=out_root)) - except OSError: - pass - _prune_semantic_cache(out_root, _live_hashes) - except Exception as exc: - print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) - stages.mark("semantic extract") - - pg_result: dict = {"nodes": [], "edges": []} - if cli_postgres_dsn is not None: - from graphify.pg_introspect import introspect_postgres - print(f"[graphify extract] introspecting PostgreSQL schema...") - try: - pg_result = introspect_postgres(cli_postgres_dsn) - except (ConnectionError, ImportError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " - f"{len(pg_result['edges'])} edges") - - cargo_result: dict = {"nodes": [], "edges": []} - if cli_cargo: - from graphify.cargo_introspect import introspect_cargo - print("[graphify extract] introspecting Cargo workspace...") - try: - cargo_result = introspect_cargo(target) - except (ConnectionError, ImportError, OSError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " - f"{len(cargo_result['edges'])} edges") - - # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST - # first means semantic node attributes win on collision (richer labels - # for symbols also referenced in docs). Hyperedges only come from the - # semantic side. - merged: dict = { - "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), - "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), - "hyperedges": list(sem_result.get("hyperedges", [])), - "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), - "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), - } - - graph_json_path = graphify_out / "graph.json" - analysis_path = graphify_out / ".graphify_analysis.json" - - # Build a manifest-safe files dict: only stamp semantic_hash for files - # that actually produced output (cache hit or fresh extraction). Files - # whose chunk failed have no source_file entry in sem_result — leaving - # their semantic_hash empty so detect_incremental re-queues them (#933). - # Path normalization against the scan root happens inside the helper - # (#1897) so fresh root-relative source_files match detect()'s - # absolute file lists. - _manifest_files = _stamped_manifest_files(files_by_type, sem_result, target, - partial_source_files=_partial_semantic_files) - - # Files dispatched this run but dropped by _stamped_manifest_files - # above (failed chunk, LLM omission, or any future exclusion) still - # carry a stale semantic_hash from a prior successful run in the - # on-disk manifest; save_manifest's seed loop would otherwise copy it - # verbatim and mask the omission (#1948). Derived from semantic_files - # — what was actually SENT to the backend this run (narrowed by the - # incremental gate and --code-only, widened by deep mode) — NOT from - # files_by_type: the full live corpus includes untouched files that - # were never dispatched, and clearing those would blank the whole - # manifest on every partial incremental run, forcing a full-corpus - # re-extraction on the next one. - _stamped_semantic = { - f for _flist in _manifest_files.values() for f in _flist - } - _cleared_semantic = {str(p) for p in semantic_files} - _stamped_semantic - - # Full-scan manifest saves prune rows for in-root files that left the - # scan corpus but still exist on disk (#1908). The corpus must be the - # RAW detect output (files_by_type), NOT the #933-stamp-filtered - # _manifest_files above — pruning to the filtered set would erase - # failed-chunk/omitted-doc rows and every doc row on --code-only runs. - _scan_corpus = ( - {f for _fl in files_by_type.values() for f in _fl} - if has_path else None - ) +def _doctor(args: list[str]) -> None: + from graphify.helix.persistence import HelixEmbeddedStore + from graphify.security import validate_store_path - if no_cluster: - # --no-cluster: dump the raw merged extraction as graph.json. - # No NetworkX, no community detection, no analysis sidecar. - # Dedupe nodes (by id) and parallel edges so the raw output matches the - # clustered path (whose DiGraph collapses both) and stays deterministic - # across modes (#1317; node dedup also collapses shared Swift module - # anchors emitted per importing file, #1327). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - from graphify.export import ( - backup_if_protected as _backup, - existing_graph_node_count as _existing_graph_node_count, - ) - if ( - incremental_mode - and not code_files - and not semantic_files - and not deleted_files - and not pg_result.get("nodes") - and not pg_result.get("edges") - and not cargo_result.get("nodes") - and not cargo_result.get("edges") - ): - # An exclusion-only change reaches this gate (excluded files - # are deliberately NOT in deleted_files, #1908) but must still - # scrub the newly-excluded sources from the raw graph (#1909). - # This path never runs build_merge, so prune in place. - if graph_stale_sources: - _n_pruned = _prune_graph_json_sources( - existing_graph_path, graph_stale_sources - ) - if _n_pruned: - print( - f"[graphify extract] pruned {_n_pruned} node(s) from " - f"{len(graph_stale_sources)} source file(s) no longer " - "in the scan (deleted or excluded)." - ) - print( - "[graphify extract] no incremental changes detected " - "(--no-cluster); outputs left untouched." - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - merged["nodes"] = _dedupe_nodes(merged["nodes"]) - merged["edges"] = _dedupe_edges(merged["edges"]) - # Backfill source_file from endpoint nodes — this raw path bypasses - # build_from_json's backfill, and semantic edges sometimes omit it (#1279). - _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} - for _e in merged["edges"]: - if not _e.get("source_file"): - _e["source_file"] = ( - _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" - ) - # RT-parity for the raw path: an incomplete build must not force a - # partial graph over a larger complete one here either. The clustered - # path gets this from to_json's #479 guard; this path never calls - # to_json, so replicate the shrink check against the existing file and - # exit before the write/manifest unless --allow-partial is set. - if _extraction_incomplete and not cli_allow_partial: - from graphify.export import MALFORMED_GRAPH as _MALFORMED_GRAPH - _existing_n = _existing_graph_node_count(graph_json_path) - _malformed = _existing_n is _MALFORMED_GRAPH - _shrinks = isinstance(_existing_n, int) and len(merged["nodes"]) < _existing_n - if _malformed or _shrinks: - _detail = ( - f"the existing {graph_json_path} is present but unparseable " - "(corrupt or a mid-write), so a shrink cannot be ruled out" - if _malformed - else f"smaller than the existing {graph_json_path} " - f"({len(merged['nodes'])} < {_existing_n} nodes)" - ) - print( - "[graphify extract] error: extraction was incomplete (an AST/" - f"semantic pass failed) and the resulting --no-cluster graph is {_detail}. " - "Refusing to overwrite a complete graph with a partial one. Re-run after " - "fixing the failures, or pass --allow-partial to overwrite anyway.", - file=sys.stderr, - ) - sys.exit(1) - _backup(graphify_out) - from graphify.paths import write_json_atomic as _write_json_atomic - _write_json_atomic(graph_json_path, merged, indent=2) - stages.mark("write") - cost = _estimate_cost( - backend, merged["input_tokens"], merged["output_tokens"] - ) - print( - f"[graphify extract] wrote {graph_json_path} — " - f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " - f"(no clustering)" - ) - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost: ${cost:.4f}" - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - # Build graph + cluster + score + write. - from graphify.build import ( - build as _build, - build_from_json as _build_from_json, - build_merge as _build_merge, - ) - from graphify.cluster import cluster as _cluster, score_all as _score_all - from graphify.export import to_json as _to_json - from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - dedup_backend = backend if dedup_llm else None - if incremental_mode: - # Prune everything the current scan no longer covers: genuinely - # deleted manifest rows, excluded-but-alive manifest rows (#1908), - # and the graph's own stale sources — which catches files that - # became excluded without ever being manifest-listed (#1909). - _prune_sources: list[str] = list(deleted_files) - for _src in list(excluded_files) + graph_stale_sources: - if _src not in _prune_sources: - _prune_sources.append(_src) - G = _build_merge( - [merged], - graph_path=existing_graph_path, - prune_sources=_prune_sources or None, - dedup=True, - dedup_llm_backend=dedup_backend, - root=target, - ) - else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) - stages.mark("build") - if G.number_of_nodes() == 0: - print( - "[graphify extract] graph is empty — extraction produced no nodes. " - "Possible causes: all files skipped, binary-only corpus, or LLM " - "returned no edges.", - file=sys.stderr, - ) - sys.exit(1) + store_path = validate_store_path(_store_arg(args)) + with HelixEmbeddedStore(store_path, read_only=True) as store: + result = store.verify() if "--deep" in args else store.verify_counts() + mode = "deep" if "--deep" in args else "counts" + print(f"Helix store OK ({mode}): {json.dumps(result, sort_keys=True)}") - communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) - stages.mark("cluster") - cohesion = _score_all(G, communities) - try: - gods = _god_nodes(G) - except Exception: - gods = [] - try: - surprises = _surprising(G, communities) - except Exception: - surprises = [] - stages.mark("analyze") - - from graphify.export import backup_if_protected as _backup - _backup(graphify_out) - # force=True bypasses the #479 shrink guard entirely. A full build - # legitimately shrinks (fuzzy dedup collapse, deleted code) so it keeps - # force=True — EXCEPT when this run's extraction was incomplete (an - # extractor pass crashed or some semantic chunks failed). Then a partial - # graph could silently overwrite a good complete one, so fall back to the - # shrink guard (force=False) unless the user opts in with --allow-partial. - # - # Both write paths are guarded: the clustered path here via to_json's - # #479 check, and the `--no-cluster` raw-dump path above via the same - # shrink check against the existing file (existing_graph_node_count). - # - # Trade-off: this reuses to_json's coarse node-count guard, not the - # source-aware _check_shrink that watch/update use. On an incremental run - # a legitimate deletion that coincides with an unrelated transient chunk - # failure can therefore be refused here — recoverable by re-running or - # passing --allow-partial (the good graph is preserved and the manifest - # is not stamped, so the retry re-extracts). - _force_write = cli_allow_partial or not _extraction_incomplete - _wrote = _to_json(G, communities, str(graph_json_path), force=_force_write) - if not _wrote: - # The shrink guard refused: this partial build is smaller than the - # existing graph. Exit before writing the manifest/marker below, which - # would otherwise stamp these files as done and make the next - # incremental run skip re-extracting them (poisoning the manifest - # against the graph we declined to write). Exit non-zero so a retry - # re-attempts. - print( - "[graphify extract] error: extraction was incomplete (an AST/semantic " - f"pass failed) and the resulting graph is smaller than the existing " - f"{graph_json_path}. Refusing to overwrite a complete graph with a " - "partial one. Re-run after fixing the failures, or pass --allow-partial " - "to overwrite anyway.", - file=sys.stderr, - ) - sys.exit(1) - stages.mark("export") - if merged.get("output_tokens", 0) > 0: - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" - ) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "tokens": { - "input": merged["input_tokens"], - "output": merged["output_tokens"], - }, - } - from graphify.paths import write_json_atomic as _wja - _wja(analysis_path, analysis, indent=2) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) - print( - f"[graphify extract] wrote {graph_json_path}: " - f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " - f"{len(communities)} communities" - ) - print(f"[graphify extract] wrote {analysis_path}") - if incremental_mode: - _excl_note = f", {len(excluded_files)} excluded" if excluded_files else "" - print( - f"[graphify extract] incremental summary: " - f"{sem_cache_hits + unchanged_total} files cached/unchanged, " - f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted{_excl_note}" - ) - elif sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost (~{backend}): ${cost:.4f}" - ) - # extract intentionally stops at graph.json + analysis; the report and - # community labels are produced by `cluster-only` (or an agent's Step 5). - # Point standalone users at it so communities get named (#1097). - print( - "[graphify extract] next: run " - f"`graphify cluster-only {graphify_out.parent}` " - "to generate GRAPH_REPORT.md and name communities" - ) - stages.total() - - elif cmd == "cache-check": - # graphify cache-check [--root ] [--mode | --deep] - # [--prompt-file ] - # Reads file paths (one per line) from , checks semantic cache. - # --mode deep (or --deep) checks the cache/semantic-deep/ namespace - # written by `extract --mode deep` instead of cache/semantic/ (#1894). - # --prompt-file names the extraction prompt the caller will use (an agent's - # references/extraction-spec.md), restricting hits to entries produced by - # that same prompt (#1939). Omitting it reads the unattributed layout, which - # cannot see entries a fingerprinted run wrote. - # Writes: - # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges - # graphify-out/.graphify_uncached.txt — paths that need extraction - # Stdout: "Cache: N hit, M miss" - from graphify.cache import check_semantic_cache - if len(sys.argv) < 3: - print("Usage: graphify cache-check [--root ] " - "[--mode | --deep] [--prompt-file ]", file=sys.stderr) - sys.exit(1) - files_from = Path(sys.argv[2]) - root = Path(".") - cache_mode: str | None = None - prompt_file: str | None = None - i = 3 - while i < len(sys.argv): - if sys.argv[i] == "--root" and i + 1 < len(sys.argv): - root = Path(sys.argv[i + 1]) - i += 2 - elif sys.argv[i] == "--mode" and i + 1 < len(sys.argv): - cache_mode = sys.argv[i + 1] - i += 2 - elif sys.argv[i].startswith("--mode="): - cache_mode = sys.argv[i].split("=", 1)[1] - i += 1 - elif sys.argv[i] == "--deep": - cache_mode = "deep" - i += 1 - elif sys.argv[i] == "--prompt-file" and i + 1 < len(sys.argv): - prompt_file = sys.argv[i + 1] - i += 2 - elif sys.argv[i].startswith("--prompt-file="): - prompt_file = sys.argv[i].split("=", 1)[1] - i += 1 - else: - i += 1 - files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] - cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( - files, root, mode=cache_mode, prompt_file=prompt_file - ) - out = root / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - if cached_nodes or cached_edges or cached_hyperedges: - (out / ".graphify_cached.json").write_text( - json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, - ensure_ascii=False), - encoding="utf-8", - ) - (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") - print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") - - elif cmd == "merge-chunks": - # graphify merge-chunks --out - # Concatenates .graphify_chunk_*.json files written by semantic subagents. - # Deduplicates nodes by id (first writer wins). Sums token counts. - import glob as _glob - if len(sys.argv) < 3: - print("Usage: graphify merge-chunks --out ", file=sys.stderr) - sys.exit(1) - out_path: Path | None = None - chunk_args: list[str] = [] - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path = Path(sys.argv[i + 1]) - i += 2 +def dispatch_command(cmd: str) -> None: + args = sys.argv[2:] + try: + if cmd == "query": + _query(args) + elif cmd == "path": + _path(args) + elif cmd == "explain": + _explain(args) + elif cmd == "affected": + from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected + + query = next((value for value in args if not value.startswith("-")), "") + depth = int(_option(args, "--depth") or 2) + relations = [ + value.split("=", 1)[1] + for value in args + if value.startswith("--relation=") + ] + for index, value in enumerate(args): + if value == "--relation" and index + 1 < len(args): + relations.append(args[index + 1]) + loaded = _loaded(args) + print(format_affected( + loaded.graph, query, + node_query=loaded.query, + relations=relations or DEFAULT_AFFECTED_RELATIONS, + depth=depth, + )) + elif cmd in {"extract", "update"}: + _update_or_extract(cmd, args) + elif cmd == "watch": + from graphify.watch import watch + + target = Path(next((value for value in args if not value.startswith("-")), ".")) + watch( + target, + debounce=float(_option(args, "--debounce") or 3.0), + retain_rollback="--retain-rollback" in args, + ) + elif cmd == "check-update": + from graphify.watch import check_update + + check_update(Path(args[0] if args else ".")) + elif cmd == "cluster-only": + from graphify.operations import recluster, reanalyze + + store = _store_arg(args, default=Path(args[0]) / _GRAPHIFY_OUT / "graph.helix" if args and not args[0].startswith("-") else None) + recluster(store) + reanalyze(store) + print(f"Reclustered {store}.") + elif cmd == "label": + from graphify.operations import relabel + + default = ( + Path(args[0]) / _GRAPHIFY_OUT / "graph.helix" + if args and not args[0].startswith("-") + else None + ) + store = _store_arg(args, default=default) + labels = relabel( + store, + backend=_option(args, "--backend"), + model=_option(args, "--model"), + missing_only="--missing-only" in args, + max_concurrency=int(_option(args, "--max-concurrency") or 4), + batch_size=int(_option(args, "--batch-size") or 100), + ) + print(f"Labeled {len(labels)} communities in {store}.") + elif cmd == "export": + _export(args) + elif cmd == "tree": + _tree(args) + elif cmd == "save-result": + _save_result(args) + elif cmd == "reflect": + _reflect(args) + elif cmd == "benchmark": + from graphify.benchmark import print_benchmark, run_benchmark + + print_benchmark(run_benchmark(str(_store_arg(args)))) + elif cmd == "global": + _global(args) + elif cmd == "rollback": + _rollback(args) + elif cmd == "doctor": + _doctor(args) + elif cmd == "prs": + from graphify.prs import cmd_prs + + cmd_prs(args) + elif cmd == "clone": + if not args: + raise ValueError("clone requires a GitHub URL") + clone_output = _option(args, "--out") + _clone_repo(args[0], _option(args, "--branch"), Path(clone_output) if clone_output else None) + elif cmd == "hook-check": + _run_hook_guard("search") + elif cmd == "hook-guard": + _run_hook_guard( + args[0] if args else "search", + strict="--strict" in args[1:], + ) + elif cmd == "hook": + from graphify import hooks + + action = args[0] if args else "install" + target = Path(args[1] if len(args) > 1 else ".") + if action == "install": + print(hooks.install(target)) + elif action == "uninstall": + print(hooks.uninstall(target)) + elif action == "status": + print(hooks.status(target)) else: - chunk_args.append(sys.argv[i]) - i += 1 - if not out_path: - print("error: --out required", file=sys.stderr) - sys.exit(1) - chunk_files: list[str] = [] - for arg in chunk_args: - expanded = _glob.glob(arg) - chunk_files.extend(sorted(expanded) if expanded else [arg]) - merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - seen_ids: set[str] = set() - valid_chunks = 0 - # These chunk files are untrusted subagent output. load_validated_... - # stats the file size BEFORE reading it (so a multi-GB chunk can't blow up - # memory), parses the JSON, and validates the security caps + the node/ - # edge id charset that blocks path traversal (#825) — the same enforcement - # the skill merge path applies. A bad chunk is skipped with a warning - # while valid siblings still merge; if every chunk is invalid, fail - # closed instead of reporting success and replacing --out with an empty - # semantic layer. Deliberately NOT wired into - # build_from_json/load_graph_json, which must keep loading valid - # pre-existing graphs. file_type is left to build's coercion (#840). - from graphify.semantic_cleanup import load_validated_semantic_fragment - for cf in chunk_files: - chunk, _chunk_errs = load_validated_semantic_fragment(Path(cf)) - if _chunk_errs: - print( - f"[graphify merge-chunks] warning: skipping invalid chunk {cf}: " - f"{'; '.join(_chunk_errs[:3])}", - file=sys.stderr, - ) - continue - valid_chunks += 1 - for n in chunk.get("nodes", []): - if n.get("id") not in seen_ids: - seen_ids.add(n["id"]) - merged["nodes"].append(n) - merged["edges"].extend(chunk.get("edges", [])) - merged["hyperedges"].extend(chunk.get("hyperedges", [])) - # Coerce token counts: a chunk is untrusted, so a non-numeric - # input_tokens/output_tokens must not abort the whole merge with a - # TypeError after other chunks already merged. - for _tok in ("input_tokens", "output_tokens"): - _v = chunk.get(_tok, 0) - merged[_tok] += _v if isinstance(_v, (int, float)) else 0 - if not valid_chunks: - print( - f"[graphify merge-chunks] error: no valid chunks to merge; " - f"refusing to write {out_path}", - file=sys.stderr, + raise ValueError("hook action must be install, uninstall, or status") + elif cmd == "cache-check": + _cache_check(args) + elif cmd == "add": + from graphify.ingest import ingest + + if not args or args[0].startswith("-"): + raise ValueError("add requires a URL") + target = Path(_option(args, "--dir") or "raw") + saved = ingest( + args[0], target, + author=_option(args, "--author"), + contributor=_option(args, "--contributor"), ) - sys.exit(1) - out_path.parent.mkdir(parents=True, exist_ok=True) - from graphify.paths import write_json_atomic as _wja - _wja(out_path, merged, ensure_ascii=False) - chunk_summary = ( - f"{valid_chunks} chunks" - if valid_chunks == len(chunk_files) - else f"{valid_chunks} of {len(chunk_files)} chunks" - ) - print( - f"Merged {chunk_summary}: {len(merged['nodes'])} nodes, {len(merged['edges'])} edges, " - f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" - ) - - elif cmd == "merge-semantic": - # graphify merge-semantic --cached --new --out - # Merges cached semantic results with freshly-extracted chunk results. - # Deduplicates nodes by id (cached entries take priority over new ones). - if len(sys.argv) < 3: - print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) - sys.exit(1) - cached_path: Path | None = None - new_path: Path | None = None - out_path2: Path | None = None - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): - cached_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): - new_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path2 = Path(sys.argv[i + 1]); i += 2 + print(f"Saved to {saved}") + elif cmd in {"diagnose", "merge-driver"}: + raise ValueError(f"{cmd} was removed with the obsolete JSON graph format") + else: + # A bare path remains shorthand for extract. + if Path(cmd).exists(): + _update_or_extract("extract", [cmd, *args]) else: - i += 1 - if not out_path2: - print("error: --out required", file=sys.stderr) - sys.exit(1) - empty: dict = {"nodes": [], "edges": [], "hyperedges": []} - cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty - new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty - seen_ids2: set[str] = set() - all_nodes: list[dict] = [] - for n in cached_data.get("nodes", []) + new_data.get("nodes", []): - if n.get("id") not in seen_ids2: - seen_ids2.add(n["id"]) - all_nodes.append(n) - merged2 = { - "nodes": all_nodes, - "edges": cached_data.get("edges", []) + new_data.get("edges", []), - "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), - } - out_path2.parent.mkdir(parents=True, exist_ok=True) - from graphify.paths import write_json_atomic as _wja - _wja(out_path2, merged2, ensure_ascii=False) - print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") - - elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): - # User ran `graphify ` directly — treat as `graphify extract `. - # Common when following the PowerShell note in README (`graphify .`) or - # copy-pasting skill invocations without the leading slash. - sys.argv.insert(2, sys.argv[1]) - sys.argv[1] = "extract" - _reenter_main() - else: - print(f"error: unknown command '{cmd}'", file=sys.stderr) - print("Run 'graphify --help' for usage.", file=sys.stderr) - sys.exit(1) + raise ValueError(f"unknown command: {cmd}") + except (ValueError, FileNotFoundError, RuntimeError, KeyError) as exc: + print(f"error: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/graphify/cluster.py b/graphify/cluster.py index 682210700..fb2195fe9 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -1,80 +1,19 @@ -"""Community detection on NetworkX graphs. Uses Leiden (graspologic) if available, falls back to Louvain (networkx). Splits oversized communities. Returns cohesion scores.""" +"""Community detection performed directly by native Helix.""" from __future__ import annotations -import contextlib -import inspect -import io -import json -import sys -import networkx as nx +from typing import Any +from graphify.helix.access import degree_map, node_attributes, node_ids -def _suppress_output(): - """Context manager to suppress stdout/stderr during library calls. - graspologic's leiden() emits ANSI escape sequences (progress bars, - colored warnings) that corrupt PowerShell 5.1's scroll buffer on - Windows (see issue #19). Redirecting stdout/stderr to devnull during - the call prevents this without losing any graphify output. - """ - return contextlib.redirect_stdout(io.StringIO()) - - -def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: - """Run community detection. Returns {node_id: community_id}. +def _partition(graph: Any, resolution: float = 1.0) -> dict[Any, int]: + from helixdb.graph import LeidenOptions - Tries Leiden (graspologic) first — best quality. - Falls back to Louvain (built into networkx) if graspologic is not installed. - - resolution > 1.0 → more, smaller communities. - resolution < 1.0 → fewer, larger communities. - - Output from graspologic is suppressed to prevent ANSI escape codes - from corrupting terminal scroll buffers on Windows PowerShell 5.1. - """ - stable = nx.Graph() - stable.add_nodes_from(sorted(G.nodes(), key=str)) - edge_rows = sorted( - G.edges(data=True), - key=lambda row: ( - str(row[0]), - str(row[1]), - json.dumps(row[2], sort_keys=True, ensure_ascii=False, default=str), - ), - ) - for src, tgt, attrs in edge_rows: - stable.add_edge(src, tgt, **attrs) - - try: - from graspologic.partition import leiden - lsig = inspect.signature(leiden).parameters - kwargs: dict = {} - if "random_seed" in lsig: - kwargs["random_seed"] = 42 - if "trials" in lsig: - kwargs["trials"] = 1 - if "resolution" in lsig: - kwargs["resolution"] = resolution - # Suppress graspologic output to prevent ANSI escape codes from - # corrupting PowerShell 5.1 scroll buffer (issue #19) - old_stderr = sys.stderr - try: - sys.stderr = io.StringIO() - with _suppress_output(): - result = leiden(stable, **kwargs) - finally: - sys.stderr = old_stderr - return result - except ImportError: - pass - - # Fallback: networkx louvain (available since networkx 2.7). - # Inspect kwargs to stay compatible across NetworkX versions — max_level - # was added in a later release and prevents hangs on large sparse graphs. - kwargs: dict = {"seed": 42, "threshold": 1e-4, "resolution": resolution} - if "max_level" in inspect.signature(nx.community.louvain_communities).parameters: - kwargs["max_level"] = 10 - communities = nx.community.louvain_communities(stable, **kwargs) - return {node: cid for cid, nodes in enumerate(communities) for node in nodes} + result = graph.to_undirected().leiden(LeidenOptions(resolution=resolution)) + return { + node_id: community_id + for community_id, record in enumerate(result.communities) + for node_id in record.node_ids + } _MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split @@ -84,7 +23,7 @@ def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: def label_communities_by_hub( - G: nx.Graph, communities: dict[int, list[str]] + G: Any, communities: dict[int, list[str]] ) -> dict[int, str]: """Deterministic, LLM-free community labels: name each community after its highest-degree member — the structural hub — so a report reads ``auth`` / @@ -96,14 +35,15 @@ def label_communities_by_hub( overrides these with richer names. """ labels: dict[int, str] = {} + degrees = degree_map(G) for cid, members in communities.items(): - present = [n for n in members if n in G] + present = [n for n in members if G.contains_node(n)] if not present: labels[cid] = f"Community {cid}" continue # highest degree wins; ties broken by node id (ascending) for determinism - hub = min(present, key=lambda n: (-G.degree(n), str(n))) - name = str(G.nodes[hub].get("label") or hub).strip() + hub = min(present, key=lambda n: (-degrees.get(n, 0), str(n))) + name = str(node_attributes(G, hub).get("label") or hub).strip() if name.endswith("()"): name = name[:-2] labels[cid] = name or f"Community {cid}" @@ -113,7 +53,7 @@ def label_communities_by_hub( def community_member_sigs(communities: dict[int, list[str]]) -> dict[int, str]: """Per-community membership fingerprints: ``{cid: sha256(sorted member ids)}``. - Persisted next to ``.graphify_labels.json`` so a later ``cluster-only`` can tell + Persisted with native community state so a later ``cluster-only`` can tell which communities actually changed since labeling. A cid whose members no longer hash the same is a different community — reusing its old (LLM) label there is the "stale label after re-scoping" bug this guards against. Deterministic; independent @@ -132,7 +72,7 @@ def community_member_sigs(communities: dict[int, list[str]]) -> dict[int, str]: def cluster( - G: nx.Graph, + G: Any, resolution: float = 1.0, exclude_hubs_percentile: float | None = None, ) -> dict[int, list[str]]: @@ -152,32 +92,36 @@ def cluster( majority-vote neighbour community afterwards. Useful for staging/utility super-hubs that inflate god-node rankings (#919). """ - if G.number_of_nodes() == 0: + if G.node_count == 0: return {} - if G.is_directed(): + if G.directed: G = G.to_undirected() - if G.number_of_edges() == 0: - return {i: [n] for i, n in enumerate(sorted(G.nodes))} + if G.edge_count == 0: + return {i: [n] for i, n in enumerate(sorted(node_ids(G), key=repr))} + + all_degrees = degree_map(G) # Compute hub exclusion set before removing anything so degree is based on full graph hub_nodes: set[str] = set() if exclude_hubs_percentile is not None: - degrees = sorted(d for _, d in G.degree()) + degrees = sorted(all_degrees.values()) if degrees: idx = max(0, int(len(degrees) * exclude_hubs_percentile / 100) - 1) threshold = degrees[idx] - hub_nodes = {n for n, d in G.degree() if d > threshold} + hub_nodes = {n for n, d in all_degrees.items() if d > threshold} # Leiden warns and drops isolates - handle them separately # Also exclude hub nodes from partitioning so they don't pull unrelated # subsystems into the same community excluded = hub_nodes - isolates = [n for n in G.nodes() if G.degree(n) == 0 and n not in excluded] - connected_nodes = [n for n in G.nodes() if G.degree(n) > 0 and n not in excluded] - connected = G.subgraph(connected_nodes) + isolates = [n for n, value in all_degrees.items() if value == 0 and n not in excluded] + connected_nodes = [ + n for n, value in all_degrees.items() if value > 0 and n not in excluded + ] + connected = G.induced_subgraph(connected_nodes) raw: dict[int, list[str]] = {} - if connected.number_of_nodes() > 0: + if connected.node_count > 0: partition = _partition(connected, resolution=resolution) for node, cid in partition.items(): raw.setdefault(cid, []).append(node) @@ -207,7 +151,7 @@ def cluster( next_cid += 1 # Split oversized communities - max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION)) + max_size = max(_MIN_SPLIT_SIZE, int(G.node_count * _MAX_COMMUNITY_FRACTION)) final_communities: list[list[str]] = [] for nodes in raw.values(): if len(nodes) > max_size: @@ -236,10 +180,10 @@ def cluster( return {i: sorted(nodes) for i, nodes in enumerate(final_communities)} -def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]: +def _split_community(G: Any, nodes: list[str]) -> list[list[str]]: """Run a second Leiden pass on a community subgraph to split it further.""" - subgraph = G.subgraph(nodes) - if subgraph.number_of_edges() == 0: + subgraph = G.induced_subgraph(nodes) + if subgraph.edge_count == 0: # No edges - split into individual nodes return [[n] for n in sorted(nodes)] try: @@ -254,18 +198,21 @@ def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]: return [sorted(nodes)] -def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float: - """Ratio of actual intra-community edges to maximum possible.""" - n = len(community_nodes) - if n <= 1: +def cohesion_score(graph: Any, community_nodes: list[Any]) -> float: + count = len(community_nodes) + if count <= 1: return 1.0 - subgraph = G.subgraph(community_nodes) - actual = subgraph.number_of_edges() - possible = n * (n - 1) / 2 - return actual / possible if possible > 0 else 0.0 + subgraph = graph.induced_subgraph(community_nodes) + actual = len({ + frozenset((edge.source, edge.target)) + for edge in subgraph.edges() + if edge.source != edge.target + }) + possible = count * (count - 1) / 2 + return round(actual / possible, 2) if possible else 0.0 -def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]: +def score_all(G: Any, communities: dict[int, list[str]]) -> dict[int, float]: return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()} diff --git a/graphify/dedup.py b/graphify/dedup.py index 79a1bfb8a..d16bd7951 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -275,7 +275,7 @@ def _report_id_collision(nid: str, survivor: dict, losers: list[dict]) -> None: f"from '{lose_file}'. An ID is derived from the source path plus the " f"entity name, so this one does not identify a single entity and the " f"dropped node is lost. To keep them distinct, run 'graphify extract' " - f"per subfolder and merge with 'graphify merge-graphs'.", + f"per subfolder and register each native store with 'graphify global add'.", file=sys.stderr, ) @@ -530,7 +530,7 @@ def deduplicate_entities( for edge in edges: e = dict(edge) # Tolerate "from"/"to" keys from LLM backends that don't follow the - # schema exactly — build_from_json normalises later but dedup runs + # schema exactly — build_from_extraction normalises later but dedup runs # first so bracket access would KeyError here (#803). # Use explicit key presence check (not `or`) so empty-string src/tgt # aren't silently replaced by the fallback key. @@ -540,7 +540,7 @@ def deduplicate_entities( continue e["source"] = remap.get(src, src) e["target"] = remap.get(tgt, tgt) - # Remove legacy keys so they don't leak into edge attrs in graph.json. + # Remove legacy keys so they don't leak into edge attrs in native graph. e.pop("from", None) e.pop("to", None) if e["source"] != e["target"]: diff --git a/graphify/detect.py b/graphify/detect.py index c2638a0ef..6da02aa07 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -1242,7 +1242,7 @@ def _wc(path: Path) -> int: # directory subtree is silently skipped). That turns a transient # PermissionError, or a directory created/deleted mid-walk (e.g. concurrent # writes racing the scan), into a partial file list and, downstream, a - # silently partial graph.json. Record and surface every skipped directory + # silently partial native graph. Record and surface every skipped directory # so an incomplete enumeration is visible rather than silent. walk_errors: list[str] = [] diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index fcb9a11cf..26d2135ad 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -9,9 +9,6 @@ from pathlib import Path from typing import Any -import networkx as nx - - _SUPPRESSION_DECL_RE = re.compile(r"^\s*(?Pseen_[A-Za-z0-9_]+)\s*[:=]") _TYPE_TUPLE_RE = re.compile(r"set\[tuple\[(?P[^\]]+)\]\]") @@ -161,8 +158,8 @@ def diagnose_extraction( max_examples: int = 5, extract_path: str | Path | None = None, ) -> dict[str, Any]: - """Summarize same-endpoint edge-collapse risk for one JSON graph/extraction dict.""" - from graphify.build import build_from_json + """Summarize same-endpoint edge-collapse risk for an extraction payload.""" + from graphify.build import build_from_extraction node_ids = _node_ids(extraction) raw_edges = _edge_list(extraction) @@ -170,7 +167,7 @@ def diagnose_extraction( # Code-typed semantic nodes the extractor could not verify against the source # it read (#1949): likely-inferred (or hallucinated) symbols surfaced from a - # document. Count them so the flag on graph.json nodes is actually surfaced. + # document. Count them so the verification flag is surfaced. unverified_node_count = sum( 1 for n in extraction.get("nodes", []) if isinstance(n, dict) and n.get("verification") == "unverified" @@ -234,10 +231,10 @@ def diagnose_extraction( post_build_node_count: int | None = None try: graph_input = deepcopy(extraction) - graph: nx.Graph = build_from_json(graph_input, directed=directed, root=root) - graph_type = type(graph).__name__ - post_build_edge_count = graph.number_of_edges() - post_build_node_count = graph.number_of_nodes() + graph = build_from_extraction(graph_input, directed=directed, root=root) + graph_type = graph.kind + post_build_edge_count = graph.edge_count + post_build_node_count = graph.node_count except Exception as exc: build_error = f"{type(exc).__name__}: {exc}" @@ -277,56 +274,6 @@ def diagnose_extraction( } -def _read_json_file(path: str | Path) -> dict[str, Any]: - """Read a JSON graph after applying Graphify's graph-load size cap.""" - from graphify.security import check_graph_file_size_cap - - json_path = Path(path) - check_graph_file_size_cap(json_path) - try: - data = json.loads(json_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - raise RuntimeError( - f"Cannot parse {json_path}: {exc}. " - "The file may be corrupted — re-run 'graphify extract'." - ) from exc - if not isinstance(data, dict): - raise ValueError("diagnostic input must be a JSON object") - return data - - -def diagnose_file( - path: str | Path, - *, - directed: bool | None = None, - root: str | Path | None = None, - max_examples: int = 5, - extract_path: str | Path | None = None, -) -> dict[str, Any]: - """Diagnose a graph/extraction JSON file without mutating it. - - When `directed` is None, the JSON's "directed" flag is honored. Raw - extraction JSON that has no "directed" flag defaults to directed analysis. - """ - data = _read_json_file(path) - if directed is None: - raw_directed = data.get("directed") - effective_directed = raw_directed if isinstance(raw_directed, bool) else True - else: - effective_directed = directed - - summary = diagnose_extraction( - data, - directed=effective_directed, - root=root, - max_examples=max_examples, - extract_path=extract_path, - ) - summary["input_path"] = str(path) - summary["effective_directed"] = effective_directed - return summary - - def format_diagnostic_json(summary: dict[str, Any]) -> dict[str, Any]: return { "schema_version": 1, @@ -339,7 +286,7 @@ def format_diagnostic_json(summary: dict[str, Any]) -> dict[str, Any]: "producer_suppression": summary.get("producer_suppression", {}), "notes": [ "Diagnostics are read-only.", - "A normal graph.json is already post-build and cannot recover raw producer edges.", + "An active Helix generation is post-build and cannot recover raw producer edges.", "Producer suppression sites are heuristic source-code evidence.", ], } @@ -350,7 +297,7 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: lines = [ "[graphify] MultiDiGraph edge-collapse diagnostic", f"input: {summary.get('input_path', '')}", - "input_stage: provided JSON (normal graph.json is post-build)", + "input_stage: extraction payload", f"effective_directed: {summary.get('effective_directed', '')}", f"nodes: {summary['node_count']}", f"unverified_code_nodes: {summary.get('unverified_node_count', 0)}", @@ -400,7 +347,5 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"locations={example['source_locations']} " f"contexts={example['contexts']}" ) - lines.append( - "note: normal graph.json is post-build; raw producer loss must be measured earlier." - ) + lines.append("note: raw producer loss must be measured before native ingestion.") return "\n".join(lines) diff --git a/graphify/export.py b/graphify/export.py index e1f2caa99..7f0569233 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -1,4 +1,4 @@ -# write graph to HTML, JSON, SVG, GraphML, Obsidian vault, and Neo4j Cypher +# write native graph snapshots to presentation and graph-database formats from __future__ import annotations import hashlib import html as _html @@ -6,13 +6,12 @@ import math import os import re -import shutil import sys from collections import Counter -from datetime import date from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph +from typing import Any + +from .helix.model import GraphBuildData, edge_attributes, graphify_attributes, node_attributes from graphify.security import sanitize_label from graphify.analyze import _node_community_map from graphify.build import edge_data @@ -20,81 +19,13 @@ from graphify.exporters.graphdb import push_to_falkordb, push_to_neo4j # noqa: E402,F401 -# Artifacts worth preserving across rebuilds (non-regenerable without LLM or curation). -_BACKUP_ARTIFACTS = [ - "graph.json", - "GRAPH_REPORT.md", - ".graphify_labels.json", - ".graphify_analysis.json", - "manifest.json", - ".graphify_semantic_marker", - "cost.json", -] - - -def backup_if_protected(out_dir: Path) -> "Path | None": - """Snapshot graph artifacts to a dated subfolder before an overwrite. +def _portable_identity(value: Any) -> str: + """Lossless string identity for formats without Helix typed IDs.""" + from helixdb.graph import external_id_to_json - Triggers when graph.json exists AND either: - - .graphify_semantic_marker is present (graph cost real LLM tokens), or - - .graphify_labels.json contains at least one non-default community label - (graph has been curated by a human or skill). - - Returns the backup folder path, or None if no backup was taken. - Never raises — backup failure prints a warning but never blocks the write. - Set GRAPHIFY_NO_BACKUP=1 to disable. - """ - if os.environ.get("GRAPHIFY_NO_BACKUP"): - return None - out = Path(out_dir) - if not (out / "graph.json").exists(): - return None - - is_semantic = (out / ".graphify_semantic_marker").exists() - is_curated = False - labels_file = out / ".graphify_labels.json" - if labels_file.exists(): - try: - labels = json.loads(labels_file.read_text(encoding="utf-8")) - is_curated = any(v != f"Community {k}" for k, v in labels.items()) - except Exception: - pass - - if not is_semantic and not is_curated: - return None - - reason = "+".join(filter(None, ["semantic" if is_semantic else "", "curated" if is_curated else ""])) - today = date.today().isoformat() - backup_dir = out / today - graph_src = out / "graph.json" - - # Skip re-copying if today's backup already has identical graph.json content. - # If content differs (graph changed since the last backup today), overwrite - # the backup in place — one folder per day, always the latest pre-overwrite state. - if backup_dir.exists() and (backup_dir / "graph.json").exists(): - src_hash = hashlib.sha256(graph_src.read_bytes()).hexdigest() - bak_hash = hashlib.sha256((backup_dir / "graph.json").read_bytes()).hexdigest() - if src_hash == bak_hash: - return backup_dir # identical content, nothing to do - - try: - backup_dir.mkdir(parents=True, exist_ok=True) - copied = 0 - for name in _BACKUP_ARTIFACTS: - src = out / name - if src.exists(): - try: - shutil.copy2(src, backup_dir / name) - copied += 1 - except Exception: - pass - if copied: - print(f"[graphify] backed up {reason} graph ({copied} files) -> {backup_dir.name}/") - return backup_dir - except Exception as exc: - import sys - print(f"[graphify] warning: backup failed ({exc}) - continuing with overwrite", file=sys.stderr) - return None + return json.dumps( + external_id_to_json(value), sort_keys=True, separators=(",", ":") + ) def _obsidian_tag(name: str) -> str: """Sanitize a community name for use as an Obsidian tag. @@ -159,15 +90,36 @@ def _yaml_str(s: str) -> str: _CONFIDENCE_SCORE_DEFAULTS = {"EXTRACTED": 1.0, "INFERRED": 0.5, "AMBIGUOUS": 0.2} -def attach_hyperedges(G: nx.Graph, hyperedges: list) -> None: +def _edge_rows(graph: Any, node_id: Any | None = None): + records = graph.edges() if node_id is None else ( + graph.edge(edge_id) for edge_id in graph.incident_edge_ids(node_id) + ) + for edge in records: + if edge is not None: + yield edge.source, edge.target, edge_attributes(edge), edge + + +def _edge_attributes(graph: Any, source: Any, target: Any) -> dict: + edge_ids = graph.edges_between(source, target) + edge = graph.edge(edge_ids[0]) if edge_ids else None + return edge_attributes(edge) if edge is not None else {} + + +def _graph_attributes(graph: Any) -> dict: + metadata = dict(graph.attributes) + value = metadata.get("graph", {}) + return dict(value) if isinstance(value, dict) else {} + + +def attach_hyperedges(G: GraphBuildData, hyperedges: list) -> None: """Store hyperedges in the graph's metadata dict.""" - existing = G.graph.get("hyperedges", []) + existing = G.attributes.get("hyperedges", []) seen_ids = {h["id"] for h in existing} for h in hyperedges: if h.get("id") and h["id"] not in seen_ids: existing.append(h) seen_ids.add(h["id"]) - G.graph["hyperedges"] = existing + G.attributes["hyperedges"] = existing def _git_head() -> str | None: @@ -180,147 +132,6 @@ def _git_head() -> str | None: return None -# Sentinel: an existing graph.json is present and non-empty but cannot be parsed -# into a node count (corrupt, mid-write, or structurally wrong). The caller must -# fail CLOSED on this — the same way to_json's #479 guard refuses to overwrite -# such a file — because we cannot prove the new graph isn't a silent shrink. -MALFORMED_GRAPH = object() - - -def existing_graph_node_count(path: "str | Path"): - """Node count of an existing graph.json. - - Returns: - - an ``int`` node count when the file parses; - - ``None`` when there is verifiably nothing to protect — absent, empty, or - over the size cap (matching how :func:`to_json` lets the new graph - replace an empty/oversized file); - - :data:`MALFORMED_GRAPH` when the file is present and non-empty but - unparseable — the caller must treat this as fail-closed (refuse to - overwrite), mirroring to_json's #479 handling of a corrupt/mid-write file. - - The raw ``--no-cluster`` write path uses this to apply the same #479 shrink - guard that :func:`to_json` applies inline for the clustered path. - """ - p = Path(path) - if not p.exists(): - return None - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(p) - except Exception: - # Oversized: reading it to compare would be the DoS the cap guards against. - return None - try: - raw = p.read_text(encoding="utf-8") - except Exception: - # Present but unreadable: fail closed if it has bytes, else nothing to lose. - try: - return MALFORMED_GRAPH if p.stat().st_size > 0 else None - except Exception: - return None - if not raw.strip(): - return None - try: - data = json.loads(raw) - except Exception: - return MALFORMED_GRAPH - nodes = data.get("nodes") if isinstance(data, dict) else None - return len(nodes) if isinstance(nodes, list) else MALFORMED_GRAPH - - -def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, force: bool = False, built_at_commit: str | None = None, community_labels: dict[int, str] | None = None) -> bool: - # Safety check: refuse to silently shrink an existing graph (#479) - existing_path = Path(output_path) - if not force and existing_path.exists(): - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(existing_path) - except Exception: - # Existing graph.json trips the size cap; reading it to compare would - # be the very DoS the cap guards against. Can't verify — let the new - # graph replace the oversized file. - oversized = True - else: - oversized = False - if not oversized: - try: - raw = existing_path.read_text(encoding="utf-8") - except Exception: - raw = "" - if not raw.strip(): - # Empty/whitespace existing file (e.g. a freshly touched path): - # no nodes to lose, so any new graph is a growth — proceed. - existing_n = 0 - else: - try: - existing_data = json.loads(raw) - existing_n = len(existing_data.get("nodes", [])) - except Exception as exc: - # Non-empty but unparseable existing graph (corrupt or a - # mid-write): we cannot verify the new graph is not a silent - # shrink. Fail SAFE — refuse rather than overwrite. A - # fail-OPEN here (the prior behavior) is the silent data-loss - # path #479 exists to prevent: a transiently unreadable - # graph.json would let a partial rebuild clobber a good one. - import sys as _sys - print( - f"[graphify] WARNING: existing {existing_path} could not be " - f"read to verify the new graph is not smaller ({exc}). " - f"Refusing to overwrite; pass force=True to override.", - file=_sys.stderr, - ) - return False - new_n = G.number_of_nodes() - if new_n < existing_n: - import sys as _sys - print( - f"[graphify] WARNING: new graph has {new_n} nodes but existing " - f"graph.json has {existing_n} (net -{existing_n - new_n}). " - f"Refusing to overwrite. Possible causes: missing chunk files from " - f"a previous session, or fuzzy dedup collapsed same-named symbols " - f"across files during an --update on an already-current graph. " - f"Run a full rebuild (/graphify .) to be safe, or pass force=True " - f"only if you have verified the reduction is legitimate.", - file=_sys.stderr, - ) - return False - - node_community = _node_community_map(communities) - _labels: dict[int, str] = {int(k): v for k, v in (community_labels or {}).items()} - try: - data = json_graph.node_link_data(G, edges="links") - except TypeError: - data = json_graph.node_link_data(G) - for node in data["nodes"]: - cid = node_community.get(node["id"]) - node["community"] = cid - if cid is not None and _labels: - node["community_name"] = _labels.get(cid, f"Community {cid}") - node["norm_label"] = _strip_diacritics(node.get("label", "")).lower() - for link in data["links"]: - if "confidence_score" not in link: - conf = link.get("confidence", "EXTRACTED") - link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) - # Restore original edge direction. Undirected NetworkX storage may - # canonicalize endpoint order, flipping `calls` and other directional - # edges in graph.json. The build path stashes the true endpoints in - # _src/_tgt for exactly this purpose (#563). - true_src = link.pop("_src", None) - true_tgt = link.pop("_tgt", None) - if true_src is not None and true_tgt is not None: - link["source"] = true_src - link["target"] = true_tgt - data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - commit = built_at_commit if built_at_commit is not None else _git_head() - if commit: - data["built_at_commit"] = commit - from graphify.paths import write_json_atomic - # Atomic write: a crash/ENOSPC mid-write must not truncate a good graph.json. - write_json_atomic(output_path, data, indent=2) - return True - - def prune_dangling_edges(graph_data: dict) -> tuple[dict, int]: """Remove edges whose source or target node is not in the node set. @@ -381,25 +192,27 @@ def _cypher_label(raw: str, fallback: str) -> str: return cleaned -def to_cypher(G: nx.Graph, output_path: str) -> None: +def to_cypher(G: Any, output_path: str) -> None: lines = ["// Neo4j Cypher import - generated by /graphify", ""] - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) label = _cypher_escape(data.get("label", node_id)) - node_id_esc = _cypher_escape(node_id) + node_id_esc = _cypher_escape(_portable_identity(node_id)) ftype = _cypher_label( (data.get("file_type", "unknown") or "unknown").capitalize(), "Entity", ) lines.append(f"MERGE (n:{ftype} {{id: '{node_id_esc}', label: '{label}'}});") lines.append("") - for u, v, data in G.edges(data=True): + for edge in G.edges(): + u, v, data = edge.source, edge.target, edge_attributes(edge) rel = _cypher_label( (data.get("relation", "RELATES_TO") or "RELATES_TO").upper(), "RELATES_TO", ) conf = _cypher_escape(data.get("confidence", "EXTRACTED")) - u_esc = _cypher_escape(u) - v_esc = _cypher_escape(v) + u_esc = _cypher_escape(_portable_identity(u)) + v_esc = _cypher_escape(_portable_identity(v)) lines.append( f"MATCH (a {{id: '{u_esc}'}}), (b {{id: '{v_esc}'}}) " f"MERGE (a)-[:{rel} {{confidence: '{conf}'}}]->(b);" @@ -429,7 +242,7 @@ def _cap_filename(s: str, limit: int = 200) -> str: return f"{truncated}_{digest}" -def _dedup_node_filenames(G: nx.Graph, safe_name) -> dict[str, str]: +def _dedup_node_filenames(G: Any, safe_name) -> dict[Any, str]: """Map each node_id to a unique note filename, appending a numeric suffix on collision. The collision set is keyed on the lowercased name so two labels differing only by case (e.g. "References" vs "references") still get distinct @@ -439,7 +252,8 @@ def _dedup_node_filenames(G: nx.Graph, safe_name) -> dict[str, str]: silently overwrites a node whose literal label is already "base_1".""" node_filenames: dict[str, str] = {} used: set[str] = set() - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) base = safe_name(data.get("label", node_id)) candidate = base n = 1 @@ -452,7 +266,7 @@ def _dedup_node_filenames(G: nx.Graph, safe_name) -> dict[str, str]: def to_obsidian( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_dir: str, community_labels: dict[int, str] | None = None, @@ -515,7 +329,7 @@ def safe_name(label: str) -> str: # Helper: compute dominant confidence for a node across all its edges def _dominant_confidence(node_id: str) -> str: confs = [] - for u, v, edata in G.edges(node_id, data=True): + for u, v, edata, _ in _edge_rows(G, node_id): confs.append(edata.get("confidence", "EXTRACTED")) if not confs: return "EXTRACTED" @@ -531,7 +345,8 @@ def _dominant_confidence(node_id: str) -> str: # Write one .md file per node node_notes_written = 0 - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) label = data.get("label", node_id) cid = node_community.get(node_id) community_name = ( @@ -571,7 +386,10 @@ def _dominant_confidence(node_id: str) -> str: neighbors = list(G.neighbors(node_id)) if neighbors: lines.append("## Connections") - for neighbor in sorted(neighbors, key=lambda n: G.nodes[n].get("label", n)): + for neighbor in sorted( + neighbors, + key=lambda n: str(node_attributes(G, n).get("label", n)), + ): edata = edge_data(G, node_id, neighbor) neighbor_label = node_filename[neighbor] relation = edata.get("relation", "") @@ -592,7 +410,8 @@ def _dominant_confidence(node_id: str) -> str: inter_community_edges: dict[int, dict[int, int]] = {} for cid in communities: inter_community_edges[cid] = {} - for u, v in G.edges(): + for edge in G.edges(): + u, v = edge.source, edge.target cu = node_community.get(u) cv = node_community.get(v) if cu is not None and cv is not None and cu != cv: @@ -642,7 +461,7 @@ def _community_name(cid) -> str: # synthesized/merge-artifact ids). Dereferencing those via G.nodes[n] or # node_filename[n] raises KeyError and aborts the whole vault export, so # skip dangling members rather than crashing (issue #1236). - members = [m for m in all_members if m in G and m in node_filename] + members = [m for m in all_members if G.contains_node(m) and m in node_filename] n_members = len(members) coh_value = cohesion.get(cid) if cohesion else None @@ -672,8 +491,10 @@ def _community_name(cid) -> str: # Members section lines.append("## Members") - for node_id in sorted(members, key=lambda n: G.nodes[n].get("label", n)): - data = G.nodes[node_id] + for node_id in sorted( + members, key=lambda n: str(node_attributes(G, n).get("label", n)) + ): + data = node_attributes(G, node_id) node_label = node_filename[node_id] ftype = data.get("file_type", "") source = data.get("source_file", "") @@ -706,7 +527,7 @@ def _community_name(cid) -> str: # Top bridge nodes - highest degree nodes that connect to other communities bridge_nodes = [ - (node_id, G.degree(node_id), _community_reach(node_id)) + (node_id, G.degree(node_id).degree, _community_reach(node_id)) for node_id in members if _community_reach(node_id) > 0 ] @@ -783,7 +604,7 @@ def _community_name(cid) -> str: def to_canvas( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_path: str, community_labels: dict[int, str] | None = None, @@ -818,8 +639,8 @@ def safe_name(label: str) -> str: # analysis sidecar) the grid below produces nothing and the canvas is written # as an empty 32-byte shell on an otherwise populated graph. Emit every node # into one synthetic community so the canvas always reflects the graph (#1324). - if not communities and G.number_of_nodes() > 0: - communities = {0: [str(n) for n in G.nodes()]} + if not communities and G.node_count > 0: + communities = {0: [node.id for node in G.nodes()]} num_communities = len(communities) cols = math.ceil(math.sqrt(num_communities)) if num_communities > 0 else 1 @@ -844,7 +665,7 @@ def safe_name(label: str) -> str: # Skip dangling community members with no backing node / filename, so box # sizing matches the cards actually laid out and `G.nodes[m]` never # KeyErrors below — mirrors the to_obsidian guard (#1236). - members = [m for m in communities[cid] if m in G and m in node_filenames] + members = [m for m in communities[cid] if G.contains_node(m) and m in node_filenames] n = len(members) inner_cols = max(1, math.ceil(math.sqrt(n))) w = max(600, 220 * inner_cols) @@ -919,16 +740,21 @@ def safe_name(label: str) -> str: inner_cols = group_cols[cid] # Same dangling-member guard as the sizing loop and to_obsidian (#1236): # a community id absent from G / node_filenames would KeyError the sort. - members = [m for m in members if m in G and m in node_filenames] - sorted_members = sorted(members, key=lambda n: G.nodes[n].get("label", n)) + members = [m for m in members if G.contains_node(m) and m in node_filenames] + sorted_members = sorted( + members, key=lambda n: str(node_attributes(G, n).get("label", n)) + ) for m_idx, node_id in enumerate(sorted_members): col = m_idx % inner_cols row = m_idx // inner_cols nx_x = gx + 20 + col * (180 + 20) nx_y = gy + 80 + row * (60 + 20) - fname = node_filenames.get(node_id, safe_name(G.nodes[node_id].get("label", node_id))) + fallback_name = safe_name( + str(node_attributes(G, node_id).get("label", node_id)) + ) + fname = node_filenames.get(node_id, fallback_name) canvas_nodes.append({ - "id": f"n_{node_id}", + "id": f"n_{_portable_identity(node_id)}", "type": "file", "file": f"{fname}.md", "x": nx_x, @@ -939,7 +765,7 @@ def safe_name(label: str) -> str: # Generate edges - only between nodes both in canvas, cap at 200 highest-weight all_edges_weighted: list[tuple[float, str, str, str]] = [] - for u, v, edata in G.edges(data=True): + for u, v, edata, _ in _edge_rows(G): if u in all_canvas_nodes and v in all_canvas_nodes: weight = edata.get("weight", 1.0) relation = edata.get("relation", "") @@ -950,9 +776,9 @@ def safe_name(label: str) -> str: all_edges_weighted.sort(key=lambda x: -x[0]) for weight, u, v, label in all_edges_weighted[:200]: canvas_edges.append({ - "id": f"e_{u}_{v}", - "fromNode": f"n_{u}", - "toNode": f"n_{v}", + "id": f"e_{_portable_identity(u)}_{_portable_identity(v)}", + "fromNode": f"n_{_portable_identity(u)}", + "toNode": f"n_{_portable_identity(v)}", "label": label, }) @@ -961,7 +787,7 @@ def safe_name(label: str) -> str: def to_graphml( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_path: str, ) -> None: @@ -970,61 +796,66 @@ def to_graphml( Community IDs are written as a node attribute so Gephi can colour by community. Edge confidence (EXTRACTED/INFERRED/AMBIGUOUS) is preserved as an edge attribute. """ - H = G.copy() + import xml.etree.ElementTree as ET + node_community = _node_community_map(communities) - for node_id in H.nodes(): - H.nodes[node_id]["community"] = node_community.get(node_id, -1) - # Drop internal markers (e.g. the AST-provenance "_origin" tag, #1116, and - # the "_src"/"_tgt" direction markers) — they are persistence/runtime details, - # not graph data, and should not leak into the exported file. - for _, attrs in H.nodes(data=True): - for k in [k for k in attrs if k.startswith("_")]: - del attrs[k] - for _, _, attrs in H.edges(data=True): - for k in [k for k in attrs if k.startswith("_")]: - del attrs[k] - # nx.write_graphml only accepts scalar attribute values: None raises, and a - # dict/list value (e.g. a per-node `metadata` dict, or the graph-level - # `hyperedges` list set by attach_hyperedges()) raises - # "GraphML does not support type as data values" (#1831). - # Coerce None -> "" and non-scalars -> a JSON string, across all three scopes. - def _graphml_safe(val): - if val is None: - return "" - if isinstance(val, bool) or isinstance(val, (int, float, str)): - return val # GraphML-native scalars pass through unchanged - try: - return json.dumps(val, default=str, sort_keys=True) - except (TypeError, ValueError): - return str(val) - - for key, val in list(H.graph.items()): - H.graph[key] = _graphml_safe(val) - for node_id in H.nodes(): - for key, val in list(H.nodes[node_id].items()): - H.nodes[node_id][key] = _graphml_safe(val) - for u, v in H.edges(): - for key, val in list(H.edges[u, v].items()): - H.edges[u, v][key] = _graphml_safe(val) - - # Write atomically: a mid-serialization error otherwise leaves a 0-byte - # .graphml on disk that downstream tooling mistakes for a completed export - # (#1831). Write to a sibling temp file, then replace on success. + root = ET.Element("graphml", xmlns="http://graphml.graphdrawing.org/xmlns") + nodes = [ + ( + node.id, + { + **{k: v for k, v in graphify_attributes(node.attributes).items() if not k.startswith("_")}, + "community": node_community.get(node.id, -1), + }, + ) + for node in G.nodes() + ] + edges = [ + ( + edge.source, + edge.target, + {k: v for k, v in edge_attributes(edge).items() if not k.startswith("_")}, + ) + for edge in G.edges() + ] + keys = sorted( + {key for _, values in nodes for key in values} + | {key for _, _, values in edges for key in values} + ) + for key in keys: + ET.SubElement(root, "key", attrib={ + "id": str(key), "for": "all", "attr.name": str(key), "attr.type": "string" + }) + graph_element = ET.SubElement( + root, "graph", edgedefault="directed" if G.directed else "undirected" + ) + for node_id, values in nodes: + element = ET.SubElement(graph_element, "node", id=_portable_identity(node_id)) + for key, value in sorted(values.items()): + data = ET.SubElement(element, "data", key=str(key)) + data.text = value if isinstance(value, str) else json.dumps(value, default=str, sort_keys=True) + for index, (source, target, values) in enumerate(edges): + element = ET.SubElement( + graph_element, + "edge", + id=f"e{index}", + source=_portable_identity(source), + target=_portable_identity(target), + ) + for key, value in sorted(values.items()): + data = ET.SubElement(element, "data", key=str(key)) + data.text = value if isinstance(value, str) else json.dumps(value, default=str, sort_keys=True) out = Path(output_path) tmp = out.with_name(out.name + ".tmp") try: - nx.write_graphml(H, str(tmp)) - os.replace(str(tmp), str(out)) + ET.ElementTree(root).write(tmp, encoding="utf-8", xml_declaration=True) + os.replace(tmp, out) finally: - if tmp.exists(): - try: - tmp.unlink() - except OSError: - pass + tmp.unlink(missing_ok=True) def to_svg( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_path: str, community_labels: dict[int, str] | None = None, @@ -1051,16 +882,23 @@ def to_svg( ax.set_facecolor("#1a1a2e") ax.axis("off") - pos = nx.spring_layout(G, seed=42, k=2.0 / (G.number_of_nodes() ** 0.5 + 1)) + from helixdb.graph import LayoutOptions - degree = dict(G.degree()) + pos = { + point.node_id: (point.x, point.y) + for point in G.spring_layout(LayoutOptions( + seed=42, k=2.0 / (G.node_count ** 0.5 + 1) + )) + } + degree = {row.node_id: int(row.degree) for row in G.degrees()} max_deg = max(degree.values(), default=1) or 1 - node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in G.nodes()] - node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in G.nodes()] + node_ids = [node.id for node in G.nodes()] + node_colors = [COMMUNITY_COLORS[node_community.get(n, 0) % len(COMMUNITY_COLORS)] for n in node_ids] + node_sizes = [300 + 1200 * (degree.get(n, 1) / max_deg) for n in node_ids] # Draw edges - dashed for non-EXTRACTED - for u, v, data in G.edges(data=True): + for u, v, data, _ in _edge_rows(G): conf = data.get("confidence", "EXTRACTED") style = "solid" if conf == "EXTRACTED" else "dashed" alpha = 0.6 if conf == "EXTRACTED" else 0.3 @@ -1069,11 +907,25 @@ def to_svg( ax.plot([x0, x1], [y0, y1], color="#aaaaaa", linewidth=0.8, linestyle=style, alpha=alpha, zorder=1) - nx.draw_networkx_nodes(G, pos, ax=ax, node_color=node_colors, - node_size=node_sizes, alpha=0.9) - nx.draw_networkx_labels(G, pos, ax=ax, - labels={n: G.nodes[n].get("label", n) for n in G.nodes()}, - font_size=7, font_color="white") + ax.scatter( + [pos[node][0] for node in node_ids], + [pos[node][1] for node in node_ids], + c=node_colors, + s=node_sizes, + alpha=0.9, + zorder=2, + ) + for node in node_ids: + ax.text( + pos[node][0], + pos[node][1], + str(node_attributes(G, node).get("label", node)), + fontsize=7, + color="white", + ha="center", + va="center", + zorder=3, + ) # Legend if community_labels: diff --git a/graphify/exporters/graphdb.py b/graphify/exporters/graphdb.py index 14c47f0d5..0b78f8c93 100644 --- a/graphify/exporters/graphdb.py +++ b/graphify/exporters/graphdb.py @@ -2,12 +2,13 @@ from __future__ import annotations from graphify.analyze import _node_community_map -import networkx as nx +from graphify.helix.model import edge_attributes, graphify_attributes import re +from typing import Any def push_to_neo4j( - G: nx.Graph, + G: Any, uri: str, user: str, password: str, @@ -42,7 +43,8 @@ def _safe_label(label: str) -> str: edges_pushed = 0 with driver.session() as session: - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) props = { k: v for k, v in data.items() if isinstance(v, (str, int, float, bool)) and not k.startswith("_") @@ -59,7 +61,8 @@ def _safe_label(label: str) -> str: ) nodes_pushed += 1 - for u, v, data in G.edges(data=True): + for edge in G.edges(): + u, v, data = edge.source, edge.target, edge_attributes(edge) rel = _safe_rel(data.get("relation", "RELATED_TO")) props = { k: v for k, v in data.items() @@ -78,7 +81,7 @@ def _safe_label(label: str) -> str: return {"nodes": nodes_pushed, "edges": edges_pushed} def push_to_falkordb( - G: nx.Graph, + G: Any, uri: str, user: str | None = None, password: str | None = None, @@ -141,7 +144,8 @@ def _safe_label(label: str) -> str: nodes_pushed = 0 edges_pushed = 0 - for node_id, data in G.nodes(data=True): + for node in G.nodes(): + node_id, data = node.id, graphify_attributes(node.attributes) props = { k: v for k, v in data.items() if isinstance(v, (str, int, float, bool)) and not k.startswith("_") @@ -157,7 +161,8 @@ def _safe_label(label: str) -> str: ) nodes_pushed += 1 - for u, v, data in G.edges(data=True): + for edge in G.edges(): + u, v, data = edge.source, edge.target, edge_attributes(edge) rel = _safe_rel(data.get("relation", "RELATED_TO")) props = { k: v for k, v in data.items() diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py index 59c0e52e3..1d7df09c6 100644 --- a/graphify/exporters/html.py +++ b/graphify/exporters/html.py @@ -6,7 +6,9 @@ import html as _html from graphify.analyze import _node_community_map import json -import networkx as nx +from typing import Any + +from graphify.helix.model import edge_attributes, graphify_attributes from graphify.security import sanitize_label @@ -323,7 +325,7 @@ def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: """ def to_html( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_path: str, community_labels: dict[int, str] | None = None, @@ -340,91 +342,34 @@ def to_html( If member_counts is provided (aggregated community view), node sizes are based on community member counts rather than graph degree. - If node_limit is set and the graph exceeds it, automatically builds an - aggregated community-level meta-graph instead of raising ValueError. + Large graphs must be exported with a non-browser format. """ limit = node_limit if node_limit is not None else _viz_node_limit() - if G.number_of_nodes() > limit: - if node_limit is not None: - # Build aggregated community meta-graph - from collections import Counter as _Counter - import networkx as _nx - print(f"Graph has {G.number_of_nodes()} nodes (above {limit} limit). Building aggregated community view...") - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - meta = _nx.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) - edge_counts = _Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, - relation=f"{w} cross-community edges", confidence="AGGREGATED") - if meta.number_of_nodes() <= 1: - print("Single community - aggregated view not useful. Skipping graph.html.") - return - meta_communities = {cid: [str(cid)] for cid in communities} - mc = {cid: len(members) for cid, members in communities.items()} - # Remap hyperedges from semantic node IDs to community IDs - raw_hyperedges = G.graph.get("hyperedges", []) - if raw_hyperedges: - remapped = [] - for he in raw_hyperedges: - he_members = he.get("nodes", []) - comm_ids, seen = [], set() - for nid in he_members: - c = node_to_community.get(nid) - if c is None: - continue - s = str(c) - if s in seen: - continue - seen.add(s) - comm_ids.append(s) - if len(comm_ids) < 2: - continue - remapped.append({ - "id": he.get("id", ""), - "label": he.get("label") or he.get("relation", "").replace("_", " "), - "nodes": comm_ids, - }) - meta.graph["hyperedges"] = remapped - to_html(meta, meta_communities, output_path, - community_labels=community_labels, member_counts=mc) - print(f"graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)") - print("Tip: run with --obsidian for full node-level detail.") - return + if G.node_count > limit: raise ValueError( - f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz " + f"Graph has {G.node_count} nodes - too large for HTML viz " f"(limit: {limit}). Use --no-viz, raise GRAPHIFY_VIZ_NODE_LIMIT, " f"or reduce input size." ) node_community = _node_community_map(communities) - degree = dict(G.degree()) + degree = {row.node_id: int(row.degree) for row in G.degrees()} max_deg = max(degree.values(), default=1) or 1 max_mc = (max(member_counts.values(), default=1) or 1) if member_counts else 1 - # Work-memory overlay (derived sidecar). When not passed explicitly, load it - # best-effort from the sibling .graphify_learning.json next to the output - # graph.html (which lives beside graph.json). Empty/missing => no learning + # Work-memory overlay comes from the active native generation. When it is not + # passed explicitly, leave it empty; graph.html remains a presentation export. # fields, so the un-annotated render is byte-identical to pre-feature. if learning_overlay is None: learning_overlay = {} - try: - from graphify.reflect import load_learning_overlay as _llo - learning_overlay = _llo(Path(output_path)) - except Exception: - learning_overlay = {} # Status -> ring color. preferred=green, contested=amber. Tentative gets no # ring (it's not yet trustworthy enough to highlight in the map). _RING = {"preferred": "#22c55e", "contested": "#f59e0b"} # Build nodes list for vis.js vis_nodes = [] - for node_id, data in G.nodes(data=True): + for record in G.nodes(): + node_id, data = record.id, graphify_attributes(record.attributes) cid = node_community.get(node_id, 0) color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] label = sanitize_label(data.get("label", node_id)) @@ -482,19 +427,15 @@ def to_html( node["title"] = _html.escape(label) + "\n" + _html.escape(sanitize_label(lesson)) vis_nodes.append(node) - # Build edges list. Restore original edge direction from _src/_tgt - # (stashed by build.py for exactly this reason): undirected NetworkX - # canonicalizes endpoint order, which would otherwise flip the arrow - # for `calls` and `rationale_for` in the rendered graph (#563). + # Build edges directly from native records, whose endpoints retain direction. vis_edges = [] - for u, v, data in G.edges(data=True): + for record in G.edges(): + u, v, data = record.source, record.target, edge_attributes(record) confidence = data.get("confidence", "EXTRACTED") relation = data.get("relation", "") - true_src = data.get("_src", u) - true_tgt = data.get("_tgt", v) vis_edges.append({ - "from": true_src, - "to": true_tgt, + "from": u, + "to": v, "label": relation, "title": _html.escape(f"{relation} [{confidence}]"), "dashes": confidence != "EXTRACTED", @@ -518,9 +459,10 @@ def _js_safe(obj) -> str: nodes_json = _js_safe(vis_nodes) edges_json = _js_safe(vis_edges) legend_json = _js_safe(legend_data) - hyperedges_json = _js_safe(getattr(G, "graph", {}).get("hyperedges", [])) + metadata = dict(G.attributes).get("graph", {}) + hyperedges_json = _js_safe(metadata.get("hyperedges", []) if isinstance(metadata, dict) else []) title = _html.escape(sanitize_label(str(output_path))) - stats = f"{G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities" + stats = f"{G.node_count} nodes · {G.edge_count} edges · {len(communities)} communities" html = f""" diff --git a/graphify/extract.py b/graphify/extract.py index a18a9b1c3..0e3b837f5 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -870,7 +870,7 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st A Swift ``import CoreKit`` names a module, not a file path, so — unlike the file-resolving JS/TS handlers — there is no existing node for the edge to point at. The returned ``(id, label)`` pairs let the extractor materialize a - ``type=module`` anchor node so the edge survives; without it ``build_from_json`` + ``type=module`` anchor node so the edge survives; without it ``build_from_extraction`` prunes every Swift import edge as a dangling/external reference (#1327). """ modules: list[tuple[str, str]] = [] @@ -1192,7 +1192,7 @@ def extract_svelte(path: Path) -> dict: existing_ids = {n["id"] for n in result.get("nodes", [])} # Source file node ID must match the one _extract_generic creates: # _make_id(str(path)) - single arg, no stem prefix. Otherwise the source - # endpoint is a phantom node and build_from_json drops the edge (#701). + # endpoint is a phantom node and build_from_extraction drops the edge (#701). file_node_id = _make_id(str(path)) aliases = _load_tsconfig_aliases(path.parent) for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src): @@ -1219,7 +1219,7 @@ def extract_svelte(path: Path) -> dict: stub_source_file = str(resolved_alias) else: # Bare/scoped import (node_modules) - use last segment; - # build_from_json drops as external if no matching node exists. + # build_from_extraction drops as external if no matching node exists. module_name = raw.split("/")[-1] if not module_name: continue @@ -2869,7 +2869,7 @@ def _key(label: str) -> str: # ── Pascal / Delphi extractor ───────────────────────────────────────────────── -# Size cap for project XML files we parse with stdlib ElementTree. +# Size cap for project XML files parsed with defusedxml. # Real .csproj/.fsproj/.vbproj/.lpk files are well under 2 MiB; anything # larger is either malformed or hostile. _PROJECT_XML_MAX_BYTES = 2 * 1024 * 1024 @@ -2878,7 +2878,7 @@ def _key(label: str) -> str: def _project_xml_is_safe(src: bytes) -> bool: """Reject XML that declares DTDs or entities. - Stdlib ``xml.etree.ElementTree`` does not cap entity expansion, so a + XML parsing also has an explicit input cap, so a crafted project file could trigger a billion-laughs style DoS. External entity resolution is already disabled by pyexpat defaults, but rejecting `` dict: - package --contains--> listed unit """ try: - import xml.etree.ElementTree as ET + from defusedxml import ElementTree as ET src = path.read_bytes() except OSError as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -3016,7 +3016,7 @@ def extract_slnx(path: Path) -> dict: Project="..."/>`` children. Unlike .sln there are no GUIDs -- projects are identified by their path. """ - import xml.etree.ElementTree as ET + from defusedxml import ElementTree as ET try: src = path.read_bytes() @@ -3095,7 +3095,7 @@ def _resolve(proj_path: str) -> str: def extract_csproj(path: Path) -> dict: """Extract packages, project refs, and target framework from a .csproj/.fsproj/.vbproj.""" - import xml.etree.ElementTree as ET + from defusedxml import ElementTree as ET try: src = path.read_bytes() @@ -3586,7 +3586,7 @@ def add_member(label: str, line_no: int, context: str) -> None: def extract_xaml(path: Path) -> dict: """Extract WPF/XAML structure, bindings, x:Class, and event handler references.""" - import xml.etree.ElementTree as ET + from defusedxml import ElementTree as ET try: src = path.read_bytes() @@ -4115,43 +4115,22 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: ProcessPoolExecutor. Args: - args: (index, path_str, root_str, cache_location_str) tuple. ``root`` - anchors hash keys / node ids / the XAML boundary; ``cache_location`` - is where the cache dir is written, decoupled per #1774. A legacy - 3-tuple (no cache_location) is still accepted for back-compat. + args: ``(index, path_str, root_str)``. Cache lookup and persistence stay + in the parent process because the cache is part of Helix state. Returns: (index, result_dict) so results can be placed back in order. """ - if len(args) == 4: - idx, path_str, root_str, cache_location_str = args - else: # legacy 3-tuple: location == anchor - idx, path_str, root_str = args - cache_location_str = root_str + idx, path_str, root_str = args[:3] path = Path(path_str) root = Path(root_str) - cache_location = Path(cache_location_str) _raise_recursion_limit() - bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES - - # Check cache first (avoid re-extraction) - if not bypass_cache: - cached = load_cached(path, root, cache_root=cache_location) - if cached is not None: - return idx, cached extractor = _get_extractor(path) if extractor is None: return idx, {"nodes": [], "edges": []} result = _safe_extract_with_xaml_root(extractor, path, root) - # Never cache a zero-node result for an extractable file. Every supported - # source produces at least a file node, so an empty node list is anomalous - # (e.g. a transient batch/parallel hiccup). Caching it makes the empty - # byte-stable across runs and silently blinds affected/explain to and - # through the file (#1666); skipping the write lets a rerun self-heal. - if not bypass_cache and "error" not in result and result.get("nodes"): - save_cached(path, result, root, cache_root=cache_location) return idx, result @@ -4161,7 +4140,7 @@ def _extract_parallel( root: Path, max_workers: int | None, total_files: int, - cache_location: Path | None = None, + cache: dict[str, dict[str, Any]] | None = None, ) -> bool: """Extract uncached files in parallel using ProcessPoolExecutor. @@ -4198,11 +4177,9 @@ def _extract_parallel( max_workers = min(max_workers, 61) max_workers = max(max_workers, 1) - # root anchors hash keys / node ids / XAML boundary; cache_location is where - # the cache dir is written (defaults to root when not decoupled) (#1774). root_str = str(root) - cache_loc_str = str(cache_location if cache_location is not None else root) - work_items = [(idx, str(path), root_str, cache_loc_str) for idx, path in uncached_work] + work_items = [(idx, str(path), root_str) for idx, path in uncached_work] + paths_by_index = {idx: path for idx, path in uncached_work} done_count = 0 _PROGRESS_INTERVAL = 100 @@ -4216,6 +4193,13 @@ def _extract_parallel( try: idx, result = future.result() per_file[idx] = result + path = paths_by_index[idx] + if ( + path.suffix not in _JS_CACHE_BYPASS_SUFFIXES + and "error" not in result + and result.get("nodes") + ): + save_cached(path, result, root, cache=cache) except Exception as exc: pos = futures[future] print( @@ -4264,7 +4248,7 @@ def _extract_sequential( per_file: list[dict | None], root: Path, total_files: int, - cache_location: Path | None = None, + cache: dict[str, dict[str, Any]] | None = None, ) -> None: """Extract uncached files sequentially (fallback for small batches).""" _PROGRESS_INTERVAL = 100 @@ -4287,7 +4271,7 @@ def _extract_sequential( result = _safe_extract_with_xaml_root(extractor, path, root) # See _extract_single_file: don't cache an anomalous zero-node result (#1666). if not bypass_cache and "error" not in result and result.get("nodes"): - save_cached(path, result, root, cache_root=cache_location) + save_cached(path, result, root, cache=cache) per_file[idx] = result if total_files >= _PROGRESS_INTERVAL: # Consistent denominator with the intermediate lines (#1693). @@ -4305,6 +4289,7 @@ def extract( root: Path | None = None, parallel: bool = True, max_workers: int | None = None, + cache: dict[str, dict[str, Any]] | None = None, ) -> dict: """Extract AST nodes and edges from a list of code files. @@ -4319,14 +4304,13 @@ def extract( symbol resolution. Pass the SCAN root whenever the cache lives somewhere else (`--out`); without it the anchor falls back to cache_root and every scanned file reads as out-of-root (#1941). - cache_root: explicit root for graphify-out/cache/ (overrides the - inferred common path prefix). Pass Path('.') when running on a - subdirectory so the cache stays at ./graphify-out/cache/. - Anchors ids/source_file only as a fallback when `root` is unset. + cache_root: deprecated compatibility anchor. No filesystem cache is + read or written. parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files, use ProcessPoolExecutor for multi-core extraction. max_workers: max subprocess count. Defaults to cpu_count (or the value of GRAPHIFY_MAX_WORKERS if set), bounded by len(uncached_work). + cache: mutable extraction-cache mapping loaded from Helix state. """ paths = [Path(p) for p in paths] anchor_root = Path(root) if root is not None else None @@ -4363,13 +4347,6 @@ def extract( root = cache_root root = root.resolve() - # #1774: the cache is an OUTPUT, so when no explicit cache_root is given it is - # written under the current working directory — never `root` (the inferred - # common parent of the inputs), which would drop graphify-out/ inside a - # read-only or foreign corpus. `root` still anchors the content-hash keys, - # node ids, symbol resolution, and the XAML project-scan boundary; only the - # cache directory's location diverges from it. - cache_location = (cache_root if cache_root is not None else Path(".")).resolve() total = len(paths) # Phase 1: separate cached hits from uncached work @@ -4382,7 +4359,7 @@ def extract( continue bypass_cache = path.suffix in _JS_CACHE_BYPASS_SUFFIXES if not bypass_cache: - cached = load_cached(path, root, cache_root=cache_location) + cached = load_cached(path, root, cache=cache) if cached is not None: per_file[i] = cached continue @@ -4393,10 +4370,10 @@ def extract( ran_parallel = False if parallel and len(uncached_work) >= _PARALLEL_THRESHOLD: ran_parallel = _extract_parallel( - uncached_work, per_file, root, max_workers, total, cache_location + uncached_work, per_file, root, max_workers, total, cache ) if not ran_parallel: - _extract_sequential(uncached_work, per_file, root, total, cache_location) + _extract_sequential(uncached_work, per_file, root, total, cache) # Fill any remaining None slots (shouldn't happen, but defensive) for i in range(total): @@ -4500,7 +4477,7 @@ def extract( _merge_decl_def_classes(all_nodes, all_edges) # Remap file node IDs from absolute-path-derived to the canonical - # {parent_dir}_{stem} spec form so (a) graph.json edge endpoints are stable + # {parent_dir}_{stem} spec form so (a) native graph edge endpoints are stable # across machines (#502) and (b) AST file nodes match the IDs semantic # subagents generate (#1033). Resolve before relativizing so paths passed in # relative form still anchor to the (resolved) root. @@ -4986,7 +4963,7 @@ def _has_import_evidence(candidate_id: str) -> bool: # Relativize source_file fields so paths are portable across machines (#555). # A target OUTSIDE the scan root (an out-of-root ProjectReference/.sln/bash # `source`) can't be made relative to root; leaving it absolute leaked the - # scan path including the OS username into a committed graph.json (#1899). + # scan path including the OS username into a committed native graph (#1899). # Fall back to a walk-up relative form, or the bare basename when that would # still embed foreign path segments (a far-away or cross-drive target). When # the node's id was itself minted from the absolute path, remap it to a @@ -5038,13 +5015,13 @@ def _portable_out_of_root_sf(p: Path) -> str: # origin_file is an internal disambiguation hint (#1462): the colliding-id pass # above reads it to keep same-named cross-file stubs distinct, after which nothing - # consumes it. Drop it from the returned nodes so it never ships into graph.json as + # consumes it. Drop it from the returned nodes so it never ships into native graph as # an absolute, machine-specific path — the same "no absolute paths in output" # contract that relativizes source_file just above (#555, #932). The per-file AST # cache keeps its own copy, which is what the colliding-id pass reads on a cache hit. for n in all_nodes: n.pop("origin_file", None) - n.pop("_callable", None) # internal indirect_call marker — never ships to graph.json + n.pop("_callable", None) # internal indirect_call marker — never ships to native graph # Tag AST provenance so the incremental watch rebuild can distinguish # AST-extracted nodes from semantic/LLM nodes. On a full re-extraction diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index e0601cc3f..2b348d5b6 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -12,7 +12,7 @@ def _csharp_namespace_id(dotted_name: str) -> str: - digest = hashlib.sha1(dotted_name.encode("utf-8")).hexdigest()[:16] + digest = hashlib.sha1(dotted_name.encode("utf-8"), usedforsecurity=False).hexdigest()[:16] return f"csharp_namespace:{digest}" REFERENCE_CONTEXTS = frozenset({ @@ -2324,7 +2324,7 @@ def walk(node, parent_class_nid: str | None = None) -> None: # Module-level import handlers (Swift) name a module, not a file # path, so there is no pre-existing node to anchor the edge to. # They return (id, label) pairs for which we materialize a - # `type=module` node; otherwise build_from_json prunes every such + # `type=module` node; otherwise build_from_extraction prunes every such # import edge as a dangling/external reference. The same module # imported from N files shares one id (file_type=code keeps # build.py validation happy; `type=module` exempts it from diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 31a2e7979..20e655453 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -628,7 +628,9 @@ def _disambiguate_colliding_node_ids( if not source_key: continue if source_key in needs_hash: - salt = hashlib.sha1(source_key.encode("utf-8")).hexdigest()[:6] + salt = hashlib.sha1( + source_key.encode("utf-8"), usedforsecurity=False + ).hexdigest()[:6] new_id = _make_id(source_key, old_id, salt) else: new_id = naive.get(source_key) or _make_id(source_key, old_id) @@ -640,7 +642,7 @@ def _disambiguate_colliding_node_ids( # No colliding ids to salt apart, but the transient `target_file` hint an # importer stamps on every resolved import (#1814) still has to be dropped # here — this early exit skips the edge loop below, so without it a - # non-colliding import would carry its absolute path into graph.json. + # non-colliding import would carry its absolute path into native storage. for edge in edges: edge.pop("target_file", None) return @@ -684,7 +686,7 @@ def _disambiguate_colliding_node_ids( # target file, key the target salt by THAT file so the salt lands on the # correct sibling. Generalizes the #1475 C/ObjC header carve-out (below) to # every language and to re_exports. `pop` it as we consume it: this is the - # hint's only reader, and its absolute path must not persist into graph.json. + # hint's only reader, and its absolute path must not persist into Helix. target_file = edge.pop("target_file", None) if target_file and edge.get("relation") in ("imports", "imports_from", "re_exports"): target_edge_key = _source_key(str(target_file), root) diff --git a/graphify/extractors/sln.py b/graphify/extractors/sln.py index 936ff9fff..76fb71b96 100644 --- a/graphify/extractors/sln.py +++ b/graphify/extractors/sln.py @@ -37,7 +37,7 @@ def extract_sln(path: Path) -> dict: # A solution folder is a VIRTUAL grouping, not a file: Visual Studio writes # its name as both the display name and the "path" (proj_name == proj_path, # no real file). Resolving it to an absolute path and keying the node id off - # that leaked the absolute scan path (incl. the OS username) into graph.json, + # that leaked the absolute scan path (incl. the OS username) into native graph, # because the CLI's id-relativization only remaps ids of real files in the # scan set — a virtual folder never matches, so its absolute id survived # (#1789). Use the folder name itself (relative, no filesystem resolution). diff --git a/graphify/global_graph.py b/graphify/global_graph.py index eddd0c92a..da0ee051a 100644 --- a/graphify/global_graph.py +++ b/graphify/global_graph.py @@ -1,184 +1,164 @@ +"""Cross-project aggregation streamed directly between embedded Helix stores.""" + from __future__ import annotations -import json -import hashlib -import sys + +import copy from datetime import datetime, timezone from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph as _jg - -_GLOBAL_DIR = Path.home() / ".graphify" -_GLOBAL_GRAPH = _GLOBAL_DIR / "global-graph.json" -_GLOBAL_MANIFEST = _GLOBAL_DIR / "global-manifest.json" - - -def _load_manifest() -> dict: - if _GLOBAL_MANIFEST.exists(): - try: - return json.loads(_GLOBAL_MANIFEST.read_text(encoding="utf-8")) - except Exception as exc: - # Don't silently wipe the user's manifest on a parse error: that - # deletes every tracked repo. Back the bad file up and surface the - # error so the user can recover or report it. - backup = _GLOBAL_MANIFEST.with_suffix( - _GLOBAL_MANIFEST.suffix + f".corrupt.{int(datetime.now(timezone.utc).timestamp())}" +from typing import Iterable + +from .helix.persistence import DEFAULT_GLOBAL_STORE, HelixEmbeddedStore +from .helix.state import new_state + + +def _project_store(path: str | Path) -> Path: + candidate = Path(path).expanduser().resolve() + if candidate.name == "graph.helix": + store = candidate + elif (candidate / "graph.helix").is_dir(): + store = candidate / "graph.helix" + else: + store = candidate / "graphify-out" / "graph.helix" + if not store.is_dir(): + if candidate.suffix.lower() == ".json" or candidate.is_file(): + raise ValueError( + "legacy JSON graphs are obsolete; rebuild the project and pass " + "its graph.helix store" ) - try: - _GLOBAL_MANIFEST.rename(backup) - print( - f"[graphify global] manifest at {_GLOBAL_MANIFEST} failed to parse ({exc}); " - f"moved to {backup} and starting fresh. Restore from the backup if this was " - f"unexpected.", - file=sys.stderr, - ) - except Exception as rename_exc: - print( - f"[graphify global] manifest at {_GLOBAL_MANIFEST} failed to parse ({exc}) " - f"and could not be backed up ({rename_exc}). Starting fresh.", - file=sys.stderr, - ) - return {"version": 1, "repos": {}} - - -def _save_manifest(manifest: dict) -> None: - _GLOBAL_DIR.mkdir(parents=True, exist_ok=True) - from graphify.paths import write_json_atomic - write_json_atomic(_GLOBAL_MANIFEST, manifest, indent=2) - - -def _load_global_graph() -> nx.Graph: - if _GLOBAL_GRAPH.exists(): - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(_GLOBAL_GRAPH) - data = json.loads(_GLOBAL_GRAPH.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - return _jg.node_link_graph(data, edges="links") - except TypeError: - return _jg.node_link_graph(data) - return nx.Graph() - - -def _save_global_graph(G: nx.Graph) -> None: - _GLOBAL_DIR.mkdir(parents=True, exist_ok=True) - try: - data = _jg.node_link_data(G, edges="links") - except TypeError: - data = _jg.node_link_data(G) - from graphify.paths import write_json_atomic - write_json_atomic(_GLOBAL_GRAPH, data, indent=2) - - -def _file_hash(path: Path) -> str: - h = hashlib.sha256() - h.update(path.read_bytes()) - return h.hexdigest()[:16] - - -def global_add(source_path: Path, repo_tag: str) -> dict: - """Add or update a project graph in the global graph. - - Returns a summary dict with keys: repo_tag, nodes_added, nodes_removed, skipped. - Skipped=True means the source graph hasn't changed since last add. - """ - from graphify.build import prefix_graph_for_global, prune_repo_from_graph - - if not source_path.exists(): - raise FileNotFoundError(f"graph not found: {source_path}") - - manifest = _load_manifest() - src_hash = _file_hash(source_path) - - existing = manifest["repos"].get(repo_tag, {}) - existing_path = existing.get("source_path", "") - if existing_path and existing_path != str(source_path.resolve()): - print( - f"[graphify global] warning: repo tag '{repo_tag}' previously pointed to " - f"{existing_path!r}, now updating to {str(source_path.resolve())!r}. " - f"Use --as to give it a different name.", - file=sys.stderr, - ) - if existing.get("source_hash") == src_hash: - return {"repo_tag": repo_tag, "nodes_added": 0, "nodes_removed": 0, "skipped": True} - - # Load source graph - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(source_path) - data = json.loads(source_path.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - src_G = _jg.node_link_graph(data, edges="links") - except TypeError: - src_G = _jg.node_link_graph(data) - - # Prefix IDs for cross-project isolation - prefixed = prefix_graph_for_global(src_G, repo_tag) - - # Load global graph and prune stale nodes for this repo - G = _load_global_graph() - removed = prune_repo_from_graph(G, repo_tag) - - # Merge external-library nodes (no source_file) by label to avoid duplication - external_labels = { - d.get("label", ""): n - for n, d in G.nodes(data=True) - if not d.get("source_file") and d.get("label") - } - # Map each deduplicated external onto the existing global node so that - # edges incident to it can be rewired instead of dropped. - remap = {} - for node, data in prefixed.nodes(data=True): - if not data.get("source_file") and data.get("label") in external_labels: - remap[node] = external_labels[data["label"]] - - # Compose: add prefixed nodes (except deduplicated externals) into global graph - for node, data in prefixed.nodes(data=True): - if node not in remap: - G.add_node(node, **data) - for u, v, data in prefixed.edges(data=True): - u = remap.get(u, u) - v = remap.get(v, v) - if u != v: # don't introduce self-loops via remapping - G.add_edge(u, v, **data) - - added = prefixed.number_of_nodes() - len(remap) - _save_global_graph(G) - - manifest["repos"][repo_tag] = { - "added_at": datetime.now(timezone.utc).isoformat(), - "source_path": str(source_path.resolve()), - "node_count": added, - "edge_count": prefixed.number_of_edges(), - "source_hash": src_hash, - } - _save_manifest(manifest) + raise FileNotFoundError(f"Helix store not found: {store}") + return store - return {"repo_tag": repo_tag, "nodes_added": added, "nodes_removed": removed, "skipped": False} +def _state() -> dict: + if not DEFAULT_GLOBAL_STORE.is_dir(): + return new_state(build={"kind": "global-aggregate"}) + with HelixEmbeddedStore(DEFAULT_GLOBAL_STORE, read_only=True) as store: + return copy.deepcopy(store.read_state()) -def global_remove(repo_tag: str) -> int: - """Remove all nodes for repo_tag from the global graph. Returns count removed.""" - from graphify.build import prune_repo_from_graph - manifest = _load_manifest() - if repo_tag not in manifest["repos"]: - raise KeyError(f"repo '{repo_tag}' not in global graph") +def _repos(state: dict) -> dict: + global_state = state.setdefault("global", {}) + return global_state.setdefault("repos", {}) + - G = _load_global_graph() - removed = prune_repo_from_graph(G, repo_tag) - _save_global_graph(G) +def _source_records(state: dict) -> list[tuple[Path, str, str]]: + records: list[tuple[Path, str, str]] = [] + for repo, value in sorted(_repos(state).items()): + if not isinstance(value, dict): + raise RuntimeError(f"global repository record {repo!r} is invalid") + source = Path(str(value.get("source_path", ""))).expanduser().resolve() + generation = value.get("source_generation") + if not source.is_dir() or not isinstance(generation, str): + raise RuntimeError( + f"global repository {repo!r} source is unavailable; re-add it" + ) + records.append((source, generation, repo)) + return records + + +def _save_state(state: dict, *, retain_rollback: bool) -> None: + with HelixEmbeddedStore( + DEFAULT_GLOBAL_STORE, retain_rollback=retain_rollback + ) as store: + store.save_aggregate_sources(_source_records(state), state) + + +def global_add( + source_path: Path, + repo_tag: str, + *, + retain_rollback: bool = False, +) -> dict: + """Add or replace a project by rebuilding the aggregate through native pages.""" + source = _project_store(source_path) + with HelixEmbeddedStore(source, read_only=True) as store: + source_generation = store.active_generation + metadata = store._metadata(source_generation) + state = _state() + repos = _repos(state) + existing = repos.get(repo_tag, {}) + if ( + existing.get("source_path") == str(source) + and existing.get("source_generation") == source_generation + ): + return { + "repo_tag": repo_tag, + "nodes_added": 0, + "nodes_removed": 0, + "skipped": True, + } + removed = int(existing.get("node_count", 0)) if isinstance(existing, dict) else 0 + repos[repo_tag] = { + "added_at": datetime.now(timezone.utc).isoformat(), + "source_path": str(source), + "source_generation": source_generation, + "node_count": int(metadata.get("node_count", 0)), + "edge_count": int(metadata.get("edge_count", 0)), + } + _save_state(state, retain_rollback=retain_rollback) + return { + "repo_tag": repo_tag, + "nodes_added": int(metadata.get("node_count", 0)), + "nodes_removed": removed, + "skipped": False, + } - del manifest["repos"][repo_tag] - _save_manifest(manifest) + +def global_remove(repo_tag: str, *, retain_rollback: bool = False) -> int: + if not DEFAULT_GLOBAL_STORE.is_dir(): + raise KeyError(f"repo '{repo_tag}' not in global graph") + state = _state() + repos = _repos(state) + if repo_tag not in repos: + raise KeyError(f"repo '{repo_tag}' not in global graph") + record = repos.pop(repo_tag) + removed = int(record.get("node_count", 0)) if isinstance(record, dict) else 0 + _save_state(state, retain_rollback=retain_rollback) return removed def global_list() -> dict: - """Return the manifest repos dict.""" - return _load_manifest().get("repos", {}) + if not DEFAULT_GLOBAL_STORE.is_dir(): + return {} + return dict(_repos(_state())) def global_path() -> Path: - return _GLOBAL_GRAPH + return DEFAULT_GLOBAL_STORE + + +def aggregate( + project_stores: Iterable[str | Path], + output: str | Path = DEFAULT_GLOBAL_STORE, + *, + retain_rollback: bool = False, +) -> Path: + """Create an aggregate through bounded public Helix reads and writes.""" + sources: list[tuple[Path, str, str]] = [] + state = new_state(build={"kind": "global-aggregate"}) + repos = _repos(state) + tags: dict[str, int] = {} + for raw_path in project_stores: + path = _project_store(raw_path) + base = path.parent.parent.name or "project" + tags[base] = tags.get(base, 0) + 1 + repo = base if tags[base] == 1 else f"{base}-{tags[base]}" + with HelixEmbeddedStore(path, read_only=True) as store: + generation = store.active_generation + metadata = store._metadata(generation) + repos[repo] = { + "source_path": str(path), + "source_generation": generation, + "node_count": int(metadata.get("node_count", 0)), + "edge_count": int(metadata.get("edge_count", 0)), + } + sources.append((path, generation, repo)) + destination = Path(output).expanduser().resolve() + with HelixEmbeddedStore( + destination, retain_rollback=retain_rollback + ) as store: + store.save_aggregate_sources(sources, state) + return destination + + +__all__ = ["aggregate", "global_add", "global_list", "global_path", "global_remove"] diff --git a/graphify/helix/__init__.py b/graphify/helix/__init__.py new file mode 100644 index 000000000..c32a6b891 --- /dev/null +++ b/graphify/helix/__init__.py @@ -0,0 +1,31 @@ +"""Native, revision-pinned embedded Helix runtime for Graphify.""" + +from .model import EdgeData, GraphBuildData, LoadedGraph, NodeData +from .native import NativeBackendUnavailable, native_backend_info +from .persistence import ( + DEFAULT_GLOBAL_STORE, + DEFAULT_PROJECT_STORE, + HelixEmbeddedStore, + HelixGraphReader, + graph_storage_exists, + load_graph, + persist_graph, + persist_graph_data, +) + +__all__ = [ + "HelixEmbeddedStore", + "HelixGraphReader", + "DEFAULT_GLOBAL_STORE", + "DEFAULT_PROJECT_STORE", + "NativeBackendUnavailable", + "EdgeData", + "GraphBuildData", + "LoadedGraph", + "NodeData", + "graph_storage_exists", + "load_graph", + "native_backend_info", + "persist_graph", + "persist_graph_data", +] diff --git a/graphify/helix/access.py b/graphify/helix/access.py new file mode 100644 index 000000000..abe1cc465 --- /dev/null +++ b/graphify/helix/access.py @@ -0,0 +1,76 @@ +"""Small projections over the native immutable Helix graph surface. + +These helpers keep Graphify's attribute conventions in one place. Callers use +native Helix records and algorithms directly. +""" + +from __future__ import annotations + +from typing import Any, Iterable + +from .model import edge_attributes, graphify_attributes, node_attributes + + +def node_ids(graph: Any) -> tuple[Any, ...]: + return tuple(node.id for node in graph.nodes()) + + +def node_rows(graph: Any) -> tuple[tuple[Any, dict[str, Any]], ...]: + return tuple((node.id, graphify_attributes(node.attributes)) for node in graph.nodes()) + + +def edge_rows( + graph: Any, node_id: Any | None = None +) -> tuple[tuple[Any, Any, dict[str, Any], Any], ...]: + records: Iterable[Any] + if node_id is None: + records = graph.edges() + else: + records = ( + edge + for edge_id in graph.incident_edge_ids(node_id) + if (edge := graph.edge(edge_id)) is not None + ) + return tuple( + (edge.source, edge.target, edge_attributes(edge), edge) + for edge in records + ) + + +def degree_map(graph: Any) -> dict[Any, int]: + return {row.node_id: int(row.degree) for row in graph.degrees()} + + +def degree(graph: Any, node_id: Any) -> int: + return int(graph.degree(node_id).degree) + + +def first_edge_attributes(graph: Any, source: Any, target: Any) -> dict[str, Any]: + edge_ids = graph.edges_between(source, target) + if not edge_ids and not graph.directed: + edge_ids = graph.edges_between(target, source) + edge = graph.edge(edge_ids[0]) if edge_ids else None + return edge_attributes(edge) if edge is not None else {} + + +def all_edge_attributes(graph: Any, source: Any, target: Any) -> list[dict[str, Any]]: + edge_ids = list(graph.edges_between(source, target)) + if not edge_ids and not graph.directed: + edge_ids = list(graph.edges_between(target, source)) + return [ + edge_attributes(edge) + for edge_id in edge_ids + if (edge := graph.edge(edge_id)) is not None + ] + + +__all__ = [ + "all_edge_attributes", + "degree", + "degree_map", + "edge_rows", + "first_edge_attributes", + "node_attributes", + "node_ids", + "node_rows", +] diff --git a/graphify/helix/model.py b/graphify/helix/model.py new file mode 100644 index 000000000..d8c6372e8 --- /dev/null +++ b/graphify/helix/model.py @@ -0,0 +1,213 @@ +"""Graphify's narrow construction and loaded-generation boundaries. + +``GraphBuildData`` is a disposable batch payload. It intentionally has no +adjacency structure or graph algorithms. ``LoadedGraph`` owns the native +immutable Helix snapshot and the durable state selected from the same +generation; it intentionally does not copy topology into Python. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Any, Iterable, Literal, Mapping + + +GraphKind = Literal["graph", "digraph", "multigraph", "multidigraph"] + + +def _identity_key(value: Any) -> str: + import json + from helixdb.graph import external_id_to_json + + return json.dumps(external_id_to_json(value), sort_keys=True, separators=(",", ":")) + + +def import_identity(value: Any) -> Any: + """Decode an explicit JSON-export identity, leaving ordinary IDs unchanged.""" + if isinstance(value, dict) and set(value) == {"__helix_external_id_v1"}: + from helixdb.graph import external_id_from_json + + return external_id_from_json(value) + return value + + +def _export_identity(value: Any) -> Any: + if isinstance(value, (bytes, tuple, frozenset)): + from helixdb.graph import external_id_to_json + + return external_id_to_json(value) + return value + + +@dataclass(frozen=True) +class NodeData: + id: Any + attributes: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class EdgeData: + source: Any + target: Any + attributes: Mapping[str, Any] = field(default_factory=dict) + key: Any | None = None + + +@dataclass +class GraphBuildData: + """Transient records awaiting one staged Helix generation write.""" + + kind: GraphKind = "graph" + nodes: list[NodeData] = field(default_factory=list) + edges: list[EdgeData] = field(default_factory=list) + attributes: dict[str, Any] = field(default_factory=dict) + extras: dict[str, Any] = field(default_factory=dict) + + @property + def directed(self) -> bool: + return self.kind in {"digraph", "multidigraph"} + + @property + def multigraph(self) -> bool: + return self.kind in {"multigraph", "multidigraph"} + + @property + def node_count(self) -> int: + return len(self.nodes) + + @property + def edge_count(self) -> int: + return len(self.edges) + + @classmethod + def from_node_link(cls, payload: Mapping[str, Any]) -> "GraphBuildData": + if not isinstance(payload, Mapping): + raise TypeError("node-link graph payload must be a mapping") + directed = bool(payload.get("directed", False)) + multigraph = bool(payload.get("multigraph", False)) + kind: GraphKind = ( + "multidigraph" if directed and multigraph else + "digraph" if directed else + "multigraph" if multigraph else + "graph" + ) + raw_nodes = payload.get("nodes", []) + raw_edges = payload.get("links", payload.get("edges", [])) + if not isinstance(raw_nodes, list) or not isinstance(raw_edges, list): + raise TypeError("node-link nodes and links must be lists") + nodes: list[NodeData] = [] + seen: set[str] = set() + for index, raw in enumerate(raw_nodes): + if not isinstance(raw, Mapping) or "id" not in raw: + raise TypeError(f"nodes[{index}] must contain an id") + node_id = import_identity(raw["id"]) + identity_key = _identity_key(node_id) + if identity_key in seen: + raise ValueError(f"duplicate node identifier at nodes[{index}]") + seen.add(identity_key) + nodes.append(NodeData(node_id, {k: v for k, v in raw.items() if k != "id"})) + edges: list[EdgeData] = [] + for index, raw in enumerate(raw_edges): + if not isinstance(raw, Mapping) or "source" not in raw or "target" not in raw: + raise TypeError(f"links[{index}] must contain source and target") + source, target = import_identity(raw["source"]), import_identity(raw["target"]) + if _identity_key(source) not in seen or _identity_key(target) not in seen: + raise ValueError(f"links[{index}] references a missing node") + edges.append(EdgeData( + source, + target, + {k: v for k, v in raw.items() if k not in {"source", "target", "key"}}, + import_identity(raw.get("key")) if multigraph else None, + )) + reserved = {"directed", "multigraph", "graph", "nodes", "links", "edges", "graphify_state"} + return cls( + kind=kind, + nodes=nodes, + edges=edges, + attributes=dict(payload.get("graph", {})), + extras={k: v for k, v in payload.items() if k not in reserved}, + ) + + def to_node_link( + self, + *, + state: Mapping[str, Any] | None = None, + tagged_identities: bool = False, + ) -> dict[str, Any]: + identity = _export_identity if tagged_identities else lambda value: value + payload: dict[str, Any] = { + "directed": self.directed, + "multigraph": self.multigraph, + "graph": dict(self.attributes), + "nodes": [{"id": identity(node.id), **dict(node.attributes)} for node in self.nodes], + "links": [], + **self.extras, + } + for edge in self.edges: + record = { + "source": identity(edge.source), + "target": identity(edge.target), + **dict(edge.attributes), + } + if self.multigraph: + record["key"] = identity(edge.key) + payload["links"].append(record) + if state is not None: + payload["graphify_state"] = dict(state) + return payload + + +@dataclass(frozen=True) +class LoadedGraph: + """One native graph snapshot and durable state from the same generation.""" + + graph: Any + generation: str + state: Mapping[str, Any] + metadata: Mapping[str, Any] + store_path: Path + query: Any | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "state", MappingProxyType(dict(self.state))) + object.__setattr__(self, "metadata", MappingProxyType(dict(self.metadata))) + + +def graphify_attributes(attributes: Mapping[str, Any]) -> dict[str, Any]: + """Return selected Graphify attributes from a native record.""" + nested = attributes.get("attrs") + return dict(nested) if isinstance(nested, Mapping) else dict(attributes) + + +def node_attributes(graph: Any, node_id: Any) -> dict[str, Any]: + node = graph.node(node_id) + if node is None: + raise KeyError(node_id) + return graphify_attributes(node.attributes) + + +def edge_attributes(edge: Any) -> dict[str, Any]: + """Project a native semantic label into a transient/output attribute DTO.""" + attributes = graphify_attributes(edge.attributes) + attributes.setdefault("relation", edge.label) + return attributes + + +def edge_records(graph: Any, edge_ids: Iterable[Any] | None = None) -> tuple[Any, ...]: + if edge_ids is None: + return graph.edges() + records = [] + for edge_id in edge_ids: + edge = graph.edge(edge_id) + if edge is not None: + records.append(edge) + return tuple(records) + + +__all__ = [ + "EdgeData", "GraphBuildData", "GraphKind", "LoadedGraph", "NodeData", + "edge_attributes", "edge_records", "graphify_attributes", "import_identity", + "node_attributes", +] diff --git a/graphify/helix/native.py b/graphify/helix/native.py new file mode 100644 index 000000000..d627f10c2 --- /dev/null +++ b/graphify/helix/native.py @@ -0,0 +1,126 @@ +"""Validated public-SDK boundary for Graphify's embedded Helix runtime.""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +import importlib.metadata +from pathlib import Path +from typing import Any + +import helixdb + + +HELIX_PACKAGE_INDEX = "https://pypi.org/project/helix-db/0.2.0b3/" +HELIX_PYTHON_VERSION = "0.2.0b3" +HELIX_EMBEDDED_DISTRIBUTION = "helix-db-embedded" +HELIX_EMBEDDED_VERSION = "0.2.0b3" +_DATABASE_NAME = "graphify" + + +class NativeBackendUnavailable(RuntimeError): + """Raised when the pinned embedded Helix SDK cannot be loaded safely.""" + + +@dataclass(frozen=True) +class NativeBackendInfo: + module: str + version: str | None + embedded_version: str + + +@dataclass(frozen=True) +class _NativeSurface: + helixdb_attrs: frozenset[str] + + +_REQUIRED = _NativeSurface( + helixdb_attrs=frozenset( + { + "Client", + "Disk", + "NodeRef", + "ShortestPathDirection", + "SourcePredicate", + "g", + "read_batch", + "write_batch", + "GraphSelection", + "GraphMetadataSelection", + "IdentitySelection", + "GraphEdgeId", + "LeidenOptions", + "NativeGraph", + } + ), +) + + +@lru_cache(maxsize=1) +def validate_native_backend() -> None: + """Validate the statically imported public SDK and matching embedded wheel.""" + try: + version = importlib.metadata.version("helix-db") + except importlib.metadata.PackageNotFoundError: + version = None + if version != HELIX_PYTHON_VERSION: + raise NativeBackendUnavailable( + "embedded Helix SDK version mismatch: " + f"expected {HELIX_PYTHON_VERSION}, got {version!r}" + ) + + missing = sorted(name for name in _REQUIRED.helixdb_attrs if not hasattr(helixdb, name)) + if missing: + raise NativeBackendUnavailable( + "helixdb is missing required public embedded SDK APIs: " + f"{', '.join(missing)}" + ) + try: + embedded_version = importlib.metadata.version(HELIX_EMBEDDED_DISTRIBUTION) + except importlib.metadata.PackageNotFoundError as exc: + raise NativeBackendUnavailable( + f"{HELIX_EMBEDDED_DISTRIBUTION}=={HELIX_EMBEDDED_VERSION} is required " + "for embedded Helix storage" + ) from exc + if embedded_version != HELIX_EMBEDDED_VERSION: + raise NativeBackendUnavailable( + "embedded Helix payload version mismatch: " + f"expected {HELIX_EMBEDDED_VERSION}, got {embedded_version!r}" + ) + + +def native_backend_info() -> NativeBackendInfo: + validate_native_backend() + return NativeBackendInfo( + module=helixdb.__name__, + version=importlib.metadata.version("helix-db"), + embedded_version=importlib.metadata.version(HELIX_EMBEDDED_DISTRIBUTION), + ) + + +def open_embedded_client(path: str | Path, *, read_only: bool = False) -> Any: + """Open the official in-process, on-disk Helix client at ``path``.""" + validate_native_backend() + root = Path(path) + if read_only: + if not root.is_dir(): + raise FileNotFoundError(f"embedded Helix store not found: {root}") + else: + root.mkdir(parents=True, exist_ok=True) + source = helixdb.Disk(str(root.resolve()), _DATABASE_NAME) + if read_only: + return helixdb.Client.embedded_reader(source) + return helixdb.Client.embedded(source) + + +__all__ = [ + "HELIX_PACKAGE_INDEX", + "HELIX_PYTHON_VERSION", + "HELIX_EMBEDDED_DISTRIBUTION", + "HELIX_EMBEDDED_VERSION", + "NativeBackendInfo", + "NativeBackendUnavailable", + "native_backend_info", + "open_embedded_client", + "validate_native_backend", +] diff --git a/graphify/helix/persistence.py b/graphify/helix/persistence.py new file mode 100644 index 000000000..de0b308b9 --- /dev/null +++ b/graphify/helix/persistence.py @@ -0,0 +1,2556 @@ +"""Persistent Graphify schema implemented on the official embedded Helix SDK.""" + +from __future__ import annotations + +import base64 +from contextlib import contextmanager +import copy +from dataclasses import dataclass +import hashlib +import json +import math +import os +from pathlib import Path +import re +import threading +import time +from typing import Any +import unicodedata +import uuid + +import helixdb +from helixdb.dsl import RepeatConfig, SourcePredicate, SubTraversal, g, read_batch +from helixdb.graph import external_id_from_json, external_id_to_json + +from .model import GraphBuildData, LoadedGraph, import_identity +from .native import open_embedded_client, validate_native_backend + + +_SCHEMA_VERSION = 6 +_NODE_LABEL = "GraphifyNode" +_META_LABEL = "GraphifyMeta" +_CONTROL_LABEL = "GraphifyControl" +_STATE_LABEL = "GraphifyState" +_EXTERNAL_KEY = "external_key" +_STORAGE_KEY = "storage_key" +_GENERATION = "graphify_generation" +_CONTROL_KEY = "control_key" +_ACTIVE_GENERATION = "active_generation" +_PREVIOUS_GENERATION = "previous_generation" +_ATTRS = "attrs" +_ORDER = "graphify_order" +_LEGACY_TARGET_KEY = "target_key" +_EDGE_KEY = "edge_key" +_NATIVE_WEIGHT = "graphify_weight" +_EDGE_CONTEXT = "graphify_context" +_SEARCH_LABEL = "search_label" +_SEARCH_TEXT = "search_text" +_WRITER_LOCK_FILE = ".graphify-writer.lock" +_WRITER_LOCK_TIMEOUT_SECONDS = 120.0 +_WRITE_CHUNK_SIZE = 1_000 +_STAGED_EDGE_WRITE_CHUNK_SIZE = 2_000 +_STATE_WRITE_CHUNK_SIZE = 256 +DEFAULT_MAX_NODES = 1_000_000 +DEFAULT_MAX_EDGES = 5_000_000 +DEFAULT_PROJECT_STORE = Path("graphify-out/graph.helix") +DEFAULT_GLOBAL_STORE = Path.home() / ".graphify" / "global-graph.helix" +_DURABLE_STATE = "graphify_state" +_STATE_TYPE = "$graphify_state_type" +_STATE_KIND = "state_kind" +_STATE_KEY = "state_key" +_STATE_PAYLOAD = "payload" +_STATE_REVISION = "state_revision" +_ACTIVE_STATE_REVISION = "active_state_revision" +_CHECKSUM_MODE = "checksum_mode" +_SPLIT_CHECKSUM_MODE = "topology-state-v1" +_STREAM_CHECKSUM_MODE = "topology-stream-v1" +_TOPOLOGY_CHECKSUM = "topology_checksum" +_STATE_KINDS = ("section", "community", "file", "cache") +_STATE_REVISION_KEYS = { + kind: f"active_{kind}_revision" for kind in _STATE_KINDS +} +_STATE_CHECKSUM_KEYS = { + kind: f"{kind}_state_checksum" for kind in _STATE_KINDS +} +_STATE_COUNT_KEYS = { + kind: f"{kind}_state_count" for kind in _STATE_KINDS +} +_SEARCH_TOKEN_RE = re.compile(r"\w+") + + +def _close_public_client(client: Any) -> None: + """Close the synchronous public SDK outside an active asyncio loop. + + The public b3 client intentionally rejects synchronous embedded calls from + an event-loop thread. Resource finalizers can run on such a thread, so move + the public ``close()`` call to a short-lived worker in that one case. + """ + try: + import asyncio + + asyncio.get_running_loop() + except RuntimeError: + client.close() + return + + errors: list[BaseException] = [] + + def close() -> None: + try: + client.close() + except BaseException as exc: # pragma: no cover - propagated below + errors.append(exc) + + worker = threading.Thread(target=close, name="graphify-helix-close") + worker.start() + worker.join() + if errors: + raise errors[0] + + +def _public_store_rebuild_message(exc: BaseException, path: Path) -> str | None: + """Translate public SDK format/index failures into a source-rebuild action.""" + detail = str(exc) + blockers = ( + "Migration required: writer migration must complete", + "Index lifecycle unavailable for secondary", + ) + if not any(blocker in detail for blocker in blockers): + return None + return ( + f"embedded Helix store at {path} was created by an incompatible public " + "runtime and cannot be upgraded safely; move that graph.helix directory " + "aside and run graphify update from source" + ) + + +class _StoreLock: + """Cross-platform process lock held for the lifetime of an embedded handle.""" + + def __init__( + self, + path: Path, + *, + shared: bool, + timeout: float = _WRITER_LOCK_TIMEOUT_SECONDS, + ) -> None: + self.path = path + self.shared = shared + self.timeout = timeout + self._stream: Any | None = None + + def acquire(self) -> None: + # Helix's embedded reader is snapshot-safe alongside a writer. Only + # serialize competing writers; reader/writer exclusion would prevent + # readers from retaining the previous active generation during staging. + if self.shared: + return + self.path.parent.mkdir(parents=True, exist_ok=True) + stream = self.path.open("rb" if self.shared else "a+b") + if not self.shared and self.path.stat().st_size == 0: + stream.write(b"\0") + stream.flush() + deadline = time.monotonic() + self.timeout + while True: + try: + self._lock(stream) + break + except (BlockingIOError, OSError) as exc: + if time.monotonic() >= deadline: + stream.close() + raise TimeoutError( + f"timed out waiting for embedded Helix store lock {self.path}" + ) from exc + time.sleep(0.05) + if not self.shared: + stream.seek(0) + stream.truncate() + stream.write(f"{os.getpid()}\n".encode("ascii")) + stream.flush() + self._stream = stream + + def _lock(self, stream: Any) -> None: + if os.name == "nt": + import msvcrt + + stream.seek(0) + mode = msvcrt.LK_NBRLCK if self.shared else msvcrt.LK_NBLCK + msvcrt.locking(stream.fileno(), mode, 1) + else: + import fcntl + + mode = fcntl.LOCK_SH if self.shared else fcntl.LOCK_EX + fcntl.flock(stream.fileno(), mode | fcntl.LOCK_NB) + + def release(self) -> None: + stream = self._stream + if stream is None: + return + try: + if os.name == "nt": + import msvcrt + + stream.seek(0) + msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + finally: + stream.close() + self._stream = None + + +def _json_value(value: Any, context: str) -> Any: + """Validate and normalize a value to the JSON types accepted by Helix.""" + try: + encoded = json.dumps(value, ensure_ascii=False, allow_nan=False) + return json.loads(encoded) + except (TypeError, ValueError) as exc: + raise TypeError(f"{context} must contain JSON-compatible values: {exc}") from exc + + +def _encode_state_value(value: Any) -> Any: + """Encode typed graph identifiers inside otherwise JSON-compatible state.""" + if isinstance(value, bytes): + return {_STATE_TYPE: "bytes", "value": base64.b64encode(value).decode("ascii")} + if isinstance(value, tuple): + return {_STATE_TYPE: "tuple", "value": [_encode_state_value(item) for item in value]} + if isinstance(value, frozenset): + return { + _STATE_TYPE: "frozenset", + "value": [_encode_state_value(item) for item in sorted(value, key=_encode_key)], + } + if isinstance(value, list): + return [_encode_state_value(item) for item in value] + if isinstance(value, dict): + if all(isinstance(key, str) for key in value): + return {key: _encode_state_value(item) for key, item in value.items()} + return { + _STATE_TYPE: "mapping", + "value": [ + [_encode_state_value(key), _encode_state_value(item)] + for key, item in value.items() + ], + } + return value + + +def _decode_state_value(value: Any) -> Any: + if isinstance(value, list): + return [_decode_state_value(item) for item in value] + if not isinstance(value, dict): + return value + kind = value.get(_STATE_TYPE) if set(value) == {_STATE_TYPE, "value"} else None + if kind is None: + return {key: _decode_state_value(item) for key, item in value.items()} + raw = value.get("value") + if kind == "bytes" and isinstance(raw, str): + return base64.b64decode(raw, validate=True) + if kind == "tuple" and isinstance(raw, list): + return tuple(_decode_state_value(item) for item in raw) + if kind == "frozenset" and isinstance(raw, list): + return frozenset(_decode_state_value(item) for item in raw) + if kind == "mapping" and isinstance(raw, list): + return { + _decode_state_value(pair[0]): _decode_state_value(pair[1]) + for pair in raw + if isinstance(pair, list) and len(pair) == 2 + } + raise RuntimeError(f"embedded Helix durable state has invalid typed value {kind!r}") + + +def _tagged_key(value: Any) -> dict[str, Any]: + """Return Helix's canonical tagged external-identity envelope.""" + return external_id_to_json(value) + + +def _encode_key(value: Any) -> str: + return json.dumps( + _tagged_key(value), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def _decode_tagged_key(value: dict[str, Any]) -> Any: + try: + return external_id_from_json(value) + except (TypeError, ValueError) as exc: + raise RuntimeError(f"embedded Helix graph identifier is invalid: {exc}") from exc + + +def _decode_key(value: str) -> Any: + try: + tagged = json.loads(value) + except (TypeError, json.JSONDecodeError) as exc: + raise RuntimeError("embedded Helix graph contains an invalid encoded identifier") from exc + if not isinstance(tagged, dict): + raise RuntimeError("embedded Helix graph identifier is not a tagged object") + return _decode_tagged_key(tagged) + + +def _decode_identity(value: Any) -> Any: + if not isinstance(value, dict): + raise RuntimeError("embedded Helix identity property is not tagged") + return _decode_tagged_key(value) + + +def _normalize_search_text(value: Any) -> str: + text = unicodedata.normalize("NFKD", str(value or "")).lower() + return "".join(character for character in text if not unicodedata.combining(character)) + + +def _search_properties(node_id: Any, attrs: dict[str, Any]) -> dict[str, str]: + """Denormalize public, searchable node fields for Helix predicates.""" + label = _normalize_search_text(attrs.get("norm_label") or attrs.get("label")) + source = _normalize_search_text(attrs.get("source_file")) + identity = _normalize_search_text(node_id) + tokenized_label = " ".join(part for part in _SEARCH_TOKEN_RE.findall(label)) + tokenized_source = " ".join(part for part in _SEARCH_TOKEN_RE.findall(source)) + return { + _SEARCH_LABEL: label, + _SEARCH_TEXT: "\0".join( + (label, tokenized_label, identity, source, tokenized_source) + ), + } + + +def _checksum(payload: dict[str, Any]) -> str: + def encode_non_json(value: Any) -> Any: + if isinstance(value, bytes): + return {"$graphify_bytes": base64.b64encode(value).decode("ascii")} + if isinstance(value, frozenset): + return {"$graphify_frozenset": sorted(_encode_key(item) for item in value)} + raise TypeError(f"cannot checksum {type(value).__name__}") + + encoded = json.dumps( + payload, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + default=encode_non_json, + ).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def _generation_checksum(topology_checksum: str, state_checksum: str) -> str: + """Bind independently verified topology and state into one generation hash.""" + return _checksum({ + _TOPOLOGY_CHECKSUM: topology_checksum, + "state_checksum": state_checksum, + }) + + +class _TopologyStreamChecksum: + """Order-stable checksum that can be produced from bounded record pages.""" + + def __init__( + self, + *, + directed: bool, + multigraph: bool, + graph: dict[str, Any], + extras: dict[str, Any], + ) -> None: + self._digest = hashlib.sha256() + self._add({ + "header": { + "directed": directed, + "multigraph": multigraph, + "graph": graph, + "extras": extras, + } + }) + + def _add(self, value: dict[str, Any]) -> None: + self._digest.update(_checksum(value).encode("ascii")) + self._digest.update(b"\n") + + def node(self, value: dict[str, Any]) -> None: + self._add({"node": value}) + + def edge(self, value: dict[str, Any]) -> None: + self._add({"edge": value}) + + def hexdigest(self) -> str: + return f"sha256:{self._digest.hexdigest()}" + + +def _state_revision(metadata: dict[str, Any], kind: str) -> str | None: + revision = metadata.get(_STATE_REVISION_KEYS[kind]) + if not isinstance(revision, str): + revision = metadata.get(_ACTIVE_STATE_REVISION) + return revision if isinstance(revision, str) else None + + +def _state_category_checksum( + records: list[tuple[str, str, Any, int]], +) -> str: + return _checksum({ + "records": [ + {"kind": kind, "key": key, "payload": payload, "order": order} + for kind, key, payload, order in records + ] + }) + + +def _combined_state_checksum(checksums: dict[str, str]) -> str: + return _checksum({"categories": checksums}) + + +def _state_category_value( + state: dict[str, Any], kind: str, generation: str +) -> Any: + if kind == "community": + return state.get("communities", []) + incremental = state.get("incremental", {}) + if not isinstance(incremental, dict): + incremental = {} + if kind == "file": + return incremental.get("files", {}) + if kind == "cache": + return incremental.get("extraction_cache", {}) + + sections: dict[str, Any] = {} + for key, value in state.items(): + if key == "communities": + if isinstance(value, list) and not value: + sections[key] = [] + continue + if key == "incremental" and isinstance(value, dict): + metadata = { + name: item + for name, item in value.items() + if name not in {"files", "extraction_cache"} + } + metadata["last_successful_generation"] = generation + if "files" in value and not value.get("files"): + metadata["files"] = {} + if "extraction_cache" in value and not value.get("extraction_cache"): + metadata["extraction_cache"] = {} + sections[key] = metadata + continue + if key == "build" and isinstance(value, dict): + build = dict(value) + build["generation"] = generation + sections[key] = build + continue + sections[key] = value + sections.setdefault("build", {"generation": generation}) + sections.setdefault( + "incremental", {"last_successful_generation": generation} + ) + return sections + + +def _properties(row: Any, context: str) -> dict[str, Any]: + if not isinstance(row, dict): + raise RuntimeError(f"embedded Helix {context} row is not a mapping") + nested = row.get("properties") + if isinstance(nested, dict): + return {**row, **nested} + return row + + +def _rows(result: Any, name: str) -> list[Any]: + if not isinstance(result, dict): + raise RuntimeError("embedded Helix query returned a non-mapping response") + value = result.get(name) + if value is None: + return [] + if not isinstance(value, list): + raise RuntimeError(f"embedded Helix query variable {name!r} is not a list") + return value + + +def _returned_node_id(result: Any, name: str) -> int: + """Read the public SDK's minimal add-node receipt.""" + rows = _rows(result, name) + if len(rows) != 1 or not isinstance(rows[0], dict): + raise RuntimeError(f"embedded Helix write receipt {name!r} is invalid") + current = rows[0].get("current") + node_id = current.get("node") if isinstance(current, dict) else None + if not isinstance(node_id, int) or isinstance(node_id, bool): + raise RuntimeError(f"embedded Helix write receipt {name!r} has no node ID") + return node_id + + +class HelixNodeQuery: + """Long-lived public reader for bounded native node predicates.""" + + def __init__( + self, + path: str | Path, + generation: str, + *, + max_candidates: int = 50_000, + ) -> None: + self.path = Path(path).expanduser().resolve() + self.generation = generation + self.max_candidates = max_candidates + self._client = open_embedded_client(self.path, read_only=True) + self._lock = threading.RLock() + self._closed = False + + def _query(self, batch: Any) -> Any: + with self._lock: + if self._closed: + raise RuntimeError("native Helix node query is closed") + return self._client.query(batch.to_query_request()) + + def _predicate(self, property_name: str, values: list[str]) -> Any | None: + normalized = list(dict.fromkeys( + value for raw in values + if (value := _normalize_search_text(raw)) + )) + if not normalized: + return None + matches = [ + SourcePredicate.contains(property_name, value) + for value in normalized + ] + match = matches[0] if len(matches) == 1 else SourcePredicate.or_(matches) + return SourcePredicate.and_(( + SourcePredicate.eq(_GENERATION, self.generation), + match, + )) + + def candidate_ids(self, values: list[str]) -> list[Any]: + """Return an order-stable, bounded superset from public predicates.""" + predicate = self._predicate(_SEARCH_TEXT, values) + if predicate is None: + return [] + batch = ( + read_batch() + .var_as( + "nodes", + g() + .n_with_label_where(_NODE_LABEL, predicate) + .limit(self.max_candidates) + .value_map(), + ) + .returning(["nodes"]) + ) + rows = [_properties(row, "search node") for row in _rows(self._query(batch), "nodes")] + rows.sort(key=lambda row: int(row.get(_ORDER, 0))) + return [ + _decode_identity(row[_EXTERNAL_KEY]) + for row in rows + if isinstance(row.get(_EXTERNAL_KEY), dict) + ] + + def document_frequencies(self, terms: list[str]) -> dict[str, int]: + """Count label matches natively without reconstructing topology.""" + normalized = list(dict.fromkeys( + value for raw in terms + if (value := _normalize_search_text(raw)) + )) + if not normalized: + return {} + batch = read_batch() + variables: list[str] = [] + for index, term in enumerate(normalized): + variable = f"count_{index}" + variables.append(variable) + predicate = SourcePredicate.and_(( + SourcePredicate.eq(_GENERATION, self.generation), + SourcePredicate.contains(_SEARCH_LABEL, term), + )) + batch = batch.var_as( + variable, + g().n_with_label_where(_NODE_LABEL, predicate).count(), + ) + result = self._query(batch.returning(variables)) + return { + term: int(result.get(variable, 0)) + for term, variable in zip(normalized, variables) + } + + def traverse_ids( + self, + seeds: list[Any], + depth: int, + *, + contexts: set[str], + ) -> list[Any]: + """Traverse context-filtered edges through the public query DSL.""" + if not seeds: + return [] + storage_keys = [ + HelixEmbeddedStore._storage_key(self.generation, _encode_key(seed)) + for seed in seeds + ] + seed_predicate = SourcePredicate.and_(( + SourcePredicate.eq(_GENERATION, self.generation), + SourcePredicate.is_in(_STORAGE_KEY, storage_keys), + )) + traversal = g().n_with_label_where(_NODE_LABEL, seed_predicate) + if depth > 0 and contexts: + edge_predicate = SourcePredicate.and_(( + SourcePredicate.eq(_GENERATION, self.generation), + SourcePredicate.is_in(_EDGE_CONTEXT, sorted(contexts)), + )) + step = ( + SubTraversal.new() + .both_e() + .where(edge_predicate) + .other_n() + .dedup() + ) + traversal = traversal.repeat( + RepeatConfig.new(step).times(depth).emit_all() + ) + batch = ( + read_batch() + .var_as( + "nodes", + traversal.dedup().limit(self.max_candidates).value_map(), + ) + .returning(["nodes"]) + ) + rows = [_properties(row, "traversed node") for row in _rows(self._query(batch), "nodes")] + rows.sort(key=lambda row: int(row.get(_ORDER, 0))) + return [ + _decode_identity(row[_EXTERNAL_KEY]) + for row in rows + if isinstance(row.get(_EXTERNAL_KEY), dict) + ] + + def close(self) -> None: + if not self._closed: + _close_public_client(self._client) + self._closed = True + + def __del__(self) -> None: + try: + self.close() + except Exception: + pass + + +@dataclass +class HelixEmbeddedStore: + """Graphify's durable graph schema on an in-process Helix ``Disk`` client.""" + + path: Path + _client: Any + _helix: Any + _read_only: bool + _closed: bool + _store_lock: _StoreLock + _max_nodes: int + _max_edges: int + _retain_rollback: bool + + def __init__( + self, + path: str | Path, + *, + read_only: bool = False, + retain_rollback: bool = False, + max_nodes: int = DEFAULT_MAX_NODES, + max_edges: int = DEFAULT_MAX_EDGES, + ) -> None: + for name, value in (("max_nodes", max_nodes), ("max_edges", max_edges)): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + self.path = Path(path).expanduser().resolve() + if read_only: + if not self.path.is_dir(): + raise FileNotFoundError(f"embedded Helix store not found: {self.path}") + else: + self.path.mkdir(parents=True, exist_ok=True) + validate_native_backend() + self._helix = helixdb + self._read_only = read_only + self._retain_rollback = bool(retain_rollback) + self._max_nodes = max_nodes + self._max_edges = max_edges + self._closed = False + self._store_lock = _StoreLock( + self.path / _WRITER_LOCK_FILE, + shared=read_only, + ) + self._store_lock.acquire() + try: + self._client = open_embedded_client(self.path, read_only=read_only) + except Exception as exc: + self._store_lock.release() + if message := _public_store_rebuild_message(exc, self.path): + raise RuntimeError(message) from exc + raise + if not read_only: + try: + self._validate_active_schema() + self._cleanup_inactive_generations() + self._cleanup_inactive_state_revisions() + except Exception as exc: + try: + _close_public_client(self._client) + finally: + self._store_lock.release() + if message := _public_store_rebuild_message(exc, self.path): + raise RuntimeError(message) from exc + raise + + def _query(self, batch: Any) -> Any: + if self._closed: + raise RuntimeError("embedded Helix store is closed") + return self._client.query(batch.to_query_request()) + + def _validate_active_schema(self) -> None: + """Reject stores written by an obsolete Graphify Helix schema. + + Production readers never reconstruct a graph in Python to upgrade it. + Rebuild the project from source when an older store is encountered. + """ + generation = self._active_generation(required=False) + if generation is None: + return + meta = self._metadata(generation) + version = meta.get("schema_version") + if version != _SCHEMA_VERSION: + raise RuntimeError( + "unsupported embedded Helix graph schema: " + f"expected {_SCHEMA_VERSION}, got {version!r}; rebuild from source" + ) + + def save(self, graph: GraphBuildData, *, state: dict[str, Any] | None = None) -> None: + self.save_data(graph.to_node_link(state=state)) + + def save_generation(self, graph: GraphBuildData, state: dict[str, Any]) -> None: + """Atomically stage topology and every durable Graphify record together.""" + self.save(graph, state=state) + + def topology_matches(self, graph: GraphBuildData) -> bool: + """Return whether build data exactly matches the active native topology.""" + generation = self._active_generation(required=False) + if generation is None: + return False + expected = self._metadata(generation).get(_TOPOLOGY_CHECKSUM) + if not isinstance(expected, str): + return False + return expected == _checksum(graph.to_node_link()) + + def save_data(self, payload: dict[str, Any]) -> None: + """Stage, verify, and atomically activate a durable graph generation.""" + self._save_data(payload, activate=True) + + def _save_data(self, payload: dict[str, Any], *, activate: bool) -> str: + if self._read_only: + raise RuntimeError("cannot write through a read-only embedded Helix store") + if not isinstance(payload, dict): + raise TypeError("node-link graph payload must be a mapping") + + directed = bool(payload.get("directed", False)) + multigraph = bool(payload.get("multigraph", False)) + graph_attrs = _json_value(payload.get("graph", {}), "graph metadata") + if not isinstance(graph_attrs, dict): + raise TypeError("node-link graph metadata must be a mapping") + + raw_nodes = payload.get("nodes", []) + if not isinstance(raw_nodes, list): + raise TypeError("node-link nodes must be a list") + nodes: list[ + tuple[str, dict[str, Any], dict[str, Any], dict[str, str]] + ] = [] + node_variables: dict[str, str] = {} + canonical_nodes: list[dict[str, Any]] = [] + for index, row in enumerate(raw_nodes): + if not isinstance(row, dict) or "id" not in row: + raise TypeError(f"node-link nodes[{index}] must be a mapping with an id") + attrs = dict(row) + node_id = import_identity(attrs.pop("id")) + encoded_id = _encode_key(node_id) + if encoded_id in node_variables: + raise ValueError(f"duplicate graph node identifier at nodes[{index}]") + attrs = _json_value(attrs, f"node-link nodes[{index}] attributes") + variable = f"node_{index}" + node_variables[encoded_id] = variable + nodes.append(( + encoded_id, + _tagged_key(node_id), + attrs, + _search_properties(node_id, attrs), + )) + canonical_nodes.append({"id": node_id, **attrs}) + + raw_edges = payload.get("links", payload.get("edges", [])) + if not isinstance(raw_edges, list): + raise TypeError("node-link links must be a list") + if len(raw_nodes) > self._max_nodes or len(raw_edges) > self._max_edges: + raise ValueError( + "graph exceeds configured embedded ingestion bounds: " + f"{len(raw_nodes)}/{len(raw_edges)} > " + f"{self._max_nodes}/{self._max_edges}" + ) + edges: list[ + tuple[str, str, str, dict[str, Any], str, dict[str, Any]] + ] = [] + canonical_edges: list[dict[str, Any]] = [] + for index, row in enumerate(raw_edges): + if not isinstance(row, dict) or "source" not in row or "target" not in row: + raise TypeError( + f"node-link links[{index}] must contain source and target" + ) + attrs = dict(row) + source = import_identity(attrs.pop("source")) + target = import_identity(attrs.pop("target")) + encoded_source = _encode_key(source) + encoded_target = _encode_key(target) + if encoded_source not in node_variables or encoded_target not in node_variables: + raise ValueError(f"node-link links[{index}] references a missing node") + key = import_identity(attrs.pop("key", None)) if multigraph else None + attrs = _json_value(attrs, f"node-link links[{index}] attributes") + encoded_edge_key = _encode_key(key) + relation = attrs.pop("relation", "related_to") + if not isinstance(relation, str) or not relation: + raise TypeError( + f"node-link links[{index}] relation must be a non-empty string" + ) + edges.append(( + encoded_source, + encoded_target, + encoded_edge_key, + _tagged_key(key), + relation, + attrs, + )) + canonical = { + "source": source, + "target": target, + "relation": relation, + **attrs, + } + if multigraph: + canonical["key"] = key + canonical_edges.append(canonical) + + reserved = {"directed", "multigraph", "graph", "nodes", "links", "edges"} + raw_extras = {key: value for key, value in payload.items() if key not in reserved} + raw_state = raw_extras.pop(_DURABLE_STATE, None) + generation = uuid.uuid4().hex + encoded_state: dict[str, Any] | None = None + if raw_state is not None: + if not isinstance(raw_state, dict): + raise TypeError("durable graph state must be a mapping") + state = copy.deepcopy(raw_state) + build_state = state.setdefault("build", {}) + incremental_state = state.setdefault("incremental", {}) + if not isinstance(build_state, dict) or not isinstance( + incremental_state, dict + ): + raise TypeError( + "durable build and incremental state sections must be mappings" + ) + build_state["generation"] = generation + incremental_state["last_successful_generation"] = generation + encoded_state = _json_value( + _encode_state_value(state), "durable graph state" + ) + extras = _json_value(raw_extras, "node-link top-level metadata") + canonical_topology = { + "directed": directed, + "multigraph": multigraph, + "graph": graph_attrs, + "nodes": canonical_nodes, + "links": canonical_edges, + **extras, + } + topology_checksum = _checksum(canonical_topology) + state_records = self._state_records(encoded_state) + category_records = { + kind: [record for record in state_records if record[0] == kind] + for kind in _STATE_KINDS + } + category_checksums = { + kind: _state_category_checksum(category_records[kind]) + for kind in _STATE_KINDS + } + state_checksum = _combined_state_checksum(category_checksums) + manifest = { + "schema_version": _SCHEMA_VERSION, + "directed": directed, + "multigraph": multigraph, + "graph": graph_attrs, + "extras": extras, + "node_count": len(nodes), + "edge_count": len(edges), + "state_record_count": len(state_records), + "state_checksum": state_checksum, + _ACTIVE_STATE_REVISION: generation, + _CHECKSUM_MODE: _SPLIT_CHECKSUM_MODE, + _TOPOLOGY_CHECKSUM: topology_checksum, + **{key: generation for key in _STATE_REVISION_KEYS.values()}, + **{ + _STATE_CHECKSUM_KEYS[kind]: category_checksums[kind] + for kind in _STATE_KINDS + }, + **{ + _STATE_COUNT_KEYS[kind]: len(category_records[kind]) + for kind in _STATE_KINDS + }, + "checksum": _generation_checksum(topology_checksum, state_checksum), + } + + previous_generation = self._active_generation(required=False) + try: + self._stage_generation( + generation, manifest, nodes, edges, encoded_state + ) + self._verify_generation_counts(generation) + if activate: + self._activate_generation( + generation, + create=previous_generation is None, + previous=previous_generation, + retain_previous=self._retain_rollback, + ) + except Exception: + try: + self._drop_generation(generation) + except Exception: + pass + raise + + if activate: + self._cleanup_inactive_generations() + return generation + + @staticmethod + def _global_identity(repo: str, node_id: Any, attrs: dict[str, Any]) -> str: + """Return a stable aggregate ID, coalescing external nodes by label.""" + label = attrs.get("label") + if not attrs.get("source_file") and label: + normalized = _normalize_search_text(label).strip() + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:24] + return f"external::{digest}" + return f"{repo}::{node_id}" + + def _source_node_pages( + self, generation: str + ) -> Any: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + offset = 0 + while True: + traversal = ( + self._helix.g() + .n_with_label_where(_NODE_LABEL, predicate) + .order_by(_ORDER, self._helix.Order.ASC) + .skip(offset) + .limit(_WRITE_CHUNK_SIZE) + .project(( + self._helix.Projection.property(_EXTERNAL_KEY), + self._helix.Projection.property(_ATTRS), + self._helix.Projection.property(_ORDER), + )) + ) + rows = _rows( + self._query( + self._helix.read_batch() + .var_as("nodes", traversal) + .returning(["nodes"]) + ), + "nodes", + ) + if not rows: + return + yield rows + if len(rows) < _WRITE_CHUNK_SIZE: + return + offset += len(rows) + + def _source_edge_pages( + self, generation: str + ) -> Any: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + offset = 0 + while True: + traversal = ( + self._helix.g() + .e_where(predicate) + .order_by(_ORDER, self._helix.Order.ASC) + .skip(offset) + .limit(_WRITE_CHUNK_SIZE) + .project(( + self._helix.Projection.from_endpoint( + _EXTERNAL_KEY, "source_external_key" + ), + self._helix.Projection.to_endpoint( + _EXTERNAL_KEY, "target_external_key" + ), + self._helix.Projection.from_endpoint( + _ATTRS, "source_attrs" + ), + self._helix.Projection.to_endpoint( + _ATTRS, "target_attrs" + ), + self._helix.Projection.property(_EDGE_KEY), + self._helix.Projection.property(_ATTRS), + self._helix.Projection.property("$label", "relation"), + self._helix.Projection.property(_ORDER), + )) + ) + rows = _rows( + self._query( + self._helix.read_batch() + .var_as("edges", traversal) + .returning(["edges"]) + ), + "edges", + ) + if not rows: + return + yield rows + if len(rows) < _WRITE_CHUNK_SIZE: + return + offset += len(rows) + + def _write_aggregate_nodes( + self, + generation: str, + rows: list[tuple[str, dict[str, Any], int]], + ) -> None: + batch = self._helix.write_batch() + returned: list[str] = [] + for local_index, (node_id, attrs, order) in enumerate(rows): + encoded_id = _encode_key(node_id) + variable = f"node_{local_index}" + returned.append(variable) + batch = batch.var_as( + variable, + self._helix.g().add_n( + _NODE_LABEL, + { + _GENERATION: generation, + _STORAGE_KEY: self._storage_key(generation, encoded_id), + _EXTERNAL_KEY: _tagged_key(node_id), + _ATTRS: attrs, + _ORDER: order, + **_search_properties(node_id, attrs), + }, + ), + ) + if returned: + self._query(batch.returning(returned)) + + def _write_aggregate_edges( + self, + generation: str, + rows: list[tuple[str, str, str, dict[str, Any], int]], + ) -> None: + batch = self._helix.write_batch() + returned: list[str] = [] + for local_index, (source, target, relation, attrs, order) in enumerate(rows): + source_var = f"source_{local_index}" + target_var = f"target_{local_index}" + edge_var = f"edge_{local_index}" + source_key = self._storage_key(generation, _encode_key(source)) + target_key = self._storage_key(generation, _encode_key(target)) + batch = batch.var_as( + source_var, + self._helix.g().n_with_label_where( + _NODE_LABEL, + self._helix.SourcePredicate.eq(_STORAGE_KEY, source_key), + ), + ).var_as( + target_var, + self._helix.g().n_with_label_where( + _NODE_LABEL, + self._helix.SourcePredicate.eq(_STORAGE_KEY, target_key), + ), + ).var_as( + edge_var, + self._helix.g() + .n(self._helix.NodeRef.var(source_var)) + .add_e( + relation, + self._helix.NodeRef.var(target_var), + { + _GENERATION: generation, + _EDGE_KEY: _tagged_key(None), + _ATTRS: attrs, + _ORDER: order, + **( + {_EDGE_CONTEXT: attrs["context"]} + if isinstance(attrs.get("context"), str) + and attrs["context"] + else {} + ), + **( + {_NATIVE_WEIGHT: float(attrs["weight"])} + if isinstance(attrs.get("weight"), (int, float)) + and not isinstance(attrs.get("weight"), bool) + and math.isfinite(float(attrs["weight"])) + else {} + ), + }, + ), + ) + returned.append(edge_var) + if returned: + self._query(batch.returning(returned)) + + def save_aggregate_sources( + self, + sources: list[tuple[Path, str, str]], + state: dict[str, Any], + ) -> None: + """Stream project generations into one staged aggregate generation. + + Source topology is read in bounded public-query pages and written + directly to Helix. No Python graph, adjacency structure, or full ID map + is constructed. + """ + if self._read_only: + raise RuntimeError("cannot write through a read-only embedded Helix store") + generation = uuid.uuid4().hex + previous_generation = self._active_generation(required=False) + checksum = _TopologyStreamChecksum( + directed=False, multigraph=False, graph={}, extras={} + ) + external_ids: set[str] = set() + node_count = 0 + edge_count = 0 + try: + for source_path, source_generation, repo in sources: + if source_path.resolve() == self.path: + raise ValueError("aggregate destination cannot also be a source") + with HelixEmbeddedStore(source_path, read_only=True) as source: + source._metadata(source_generation) + for page in source._source_node_pages(source_generation): + output: list[tuple[str, dict[str, Any], int]] = [] + for raw in page: + row = _properties(raw, "aggregate source node") + old_id = _decode_identity(row.get(_EXTERNAL_KEY)) + attrs = row.get(_ATTRS) + if not isinstance(attrs, dict): + raise RuntimeError( + "aggregate source node has invalid attributes" + ) + node_id = self._global_identity(repo, old_id, attrs) + if node_id.startswith("external::"): + if node_id in external_ids: + continue + external_ids.add(node_id) + projected = dict(attrs) + projected["repo"] = repo + projected.setdefault("local_id", old_id) + output.append((node_id, projected, node_count)) + checksum.node({"id": node_id, **projected}) + node_count += 1 + self._write_aggregate_nodes(generation, output) + for page in source._source_edge_pages(source_generation): + output_edges: list[ + tuple[str, str, str, dict[str, Any], int] + ] = [] + for raw in page: + row = _properties(raw, "aggregate source edge") + source_attrs = row.get("source_attrs") + target_attrs = row.get("target_attrs") + attrs = row.get(_ATTRS) + relation = row.get("relation") + if ( + not isinstance(source_attrs, dict) + or not isinstance(target_attrs, dict) + or not isinstance(attrs, dict) + or not isinstance(relation, str) + ): + raise RuntimeError( + "aggregate source edge has invalid schema fields" + ) + old_source = _decode_identity( + row.get("source_external_key") + ) + old_target = _decode_identity( + row.get("target_external_key") + ) + source_id = self._global_identity( + repo, old_source, source_attrs + ) + target_id = self._global_identity( + repo, old_target, target_attrs + ) + if source_id == target_id: + continue + projected_edge = dict(attrs) + output_edges.append(( + source_id, + target_id, + relation, + projected_edge, + edge_count, + )) + checksum.edge({ + "source": source_id, + "target": target_id, + "relation": relation, + **projected_edge, + }) + edge_count += 1 + self._write_aggregate_edges(generation, output_edges) + + encoded_state_value = copy.deepcopy(state) + build_state = encoded_state_value.setdefault("build", {}) + incremental_state = encoded_state_value.setdefault("incremental", {}) + if not isinstance(build_state, dict) or not isinstance( + incremental_state, dict + ): + raise TypeError("durable build and incremental state must be mappings") + build_state["generation"] = generation + incremental_state["last_successful_generation"] = generation + encoded_state = _json_value( + _encode_state_value(encoded_state_value), "durable graph state" + ) + state_records = self._state_records(encoded_state) + category_records = { + kind: [record for record in state_records if record[0] == kind] + for kind in _STATE_KINDS + } + category_checksums = { + kind: _state_category_checksum(category_records[kind]) + for kind in _STATE_KINDS + } + state_checksum = _combined_state_checksum(category_checksums) + topology_checksum = checksum.hexdigest() + manifest = { + "schema_version": _SCHEMA_VERSION, + "directed": False, + "multigraph": False, + "graph": {}, + "extras": {}, + "node_count": node_count, + "edge_count": edge_count, + "state_record_count": len(state_records), + "state_checksum": state_checksum, + _ACTIVE_STATE_REVISION: generation, + _CHECKSUM_MODE: _STREAM_CHECKSUM_MODE, + _TOPOLOGY_CHECKSUM: topology_checksum, + **{key: generation for key in _STATE_REVISION_KEYS.values()}, + **{ + _STATE_CHECKSUM_KEYS[kind]: category_checksums[kind] + for kind in _STATE_KINDS + }, + **{ + _STATE_COUNT_KEYS[kind]: len(category_records[kind]) + for kind in _STATE_KINDS + }, + "checksum": _generation_checksum(topology_checksum, state_checksum), + _GENERATION: generation, + } + self._query( + self._helix.write_batch() + .var_as("meta", self._helix.g().add_n(_META_LABEL, manifest)) + .returning(["meta"]) + ) + self._write_state_records(generation, state_records, generation) + self._verify_generation_counts(generation) + self._activate_generation( + generation, + create=previous_generation is None, + previous=previous_generation, + retain_previous=self._retain_rollback, + ) + except Exception: + try: + self._drop_generation(generation) + except Exception: + pass + raise + self._cleanup_inactive_generations() + + @contextmanager + def staged_graph(self, graph: GraphBuildData): + """Yield one inactive native snapshot that can be finalized in place.""" + generation = self._save_data(graph.to_node_link(), activate=False) + staged = self.load_generation(generation, attach_query=False) + try: + yield staged + finally: + if self._active_generation(required=False) != generation: + self._drop_generation(generation) + + def activate_staged(self, staged: LoadedGraph, state: dict[str, Any]) -> LoadedGraph: + """Attach durable state, verify, and activate an existing staged topology.""" + if self._read_only: + raise RuntimeError("cannot activate through a read-only embedded Helix store") + generation = staged.generation + if staged.store_path != self.path: + raise ValueError("staged graph belongs to a different embedded Helix store") + if self._metadata(generation).get("state_record_count") != 0: + raise RuntimeError("staged Helix generation has already been finalized") + + encoded = copy.deepcopy(state) + build_state = encoded.setdefault("build", {}) + incremental_state = encoded.setdefault("incremental", {}) + if not isinstance(build_state, dict) or not isinstance(incremental_state, dict): + raise TypeError("durable build and incremental state must be mappings") + build_state["generation"] = generation + incremental_state["last_successful_generation"] = generation + encoded_state = _json_value( + _encode_state_value(encoded), "durable graph state" + ) + records = self._state_records(encoded_state) + category_records = { + kind: [record for record in records if record[0] == kind] + for kind in _STATE_KINDS + } + category_checksums = { + kind: _state_category_checksum(category_records[kind]) + for kind in _STATE_KINDS + } + previous_generation = self._active_generation(required=False) + try: + meta = self._metadata(generation) + topology_checksum = meta.get(_TOPOLOGY_CHECKSUM) + if not isinstance(topology_checksum, str): + raise RuntimeError("embedded Helix metadata has no topology checksum") + self._write_state_records(generation, records, generation) + written = self._state_records_from_rows( + self._read_state_rows(generation, revision=generation) + ) + state_checksum = _combined_state_checksum(category_checksums) + if written != records: + raise RuntimeError("embedded Helix staged state failed checksum verification") + updates = { + "state_record_count": len(records), + "state_checksum": state_checksum, + "checksum": _generation_checksum(topology_checksum, state_checksum), + **{ + _STATE_CHECKSUM_KEYS[kind]: category_checksums[kind] + for kind in _STATE_KINDS + }, + **{ + _STATE_COUNT_KEYS[kind]: len(category_records[kind]) + for kind in _STATE_KINDS + }, + } + traversal = self._helix.g().n_with_label_where( + _META_LABEL, + self._helix.SourcePredicate.eq(_GENERATION, generation), + ) + for key, value in updates.items(): + traversal = traversal.set_property(key, value) + self._query( + self._helix.write_batch() + .var_as("finalize", traversal) + .returning(["finalize"]) + ) + self._verify_generation_counts(generation) + self._activate_generation( + generation, + create=previous_generation is None, + previous=previous_generation, + retain_previous=self._retain_rollback, + ) + except Exception: + self._drop_generation(generation) + raise + self._cleanup_inactive_generations() + return self.load_generation(generation) + + def replace_state( + self, + state: dict[str, Any], + *, + previous_state: dict[str, Any] | None = None, + snapshot: LoadedGraph | None = None, + ) -> None: + """Atomically replace only changed native-state categories.""" + if self._read_only: + raise RuntimeError("cannot write through a read-only embedded Helix store") + if snapshot is not None: + if snapshot.store_path != self.path: + raise ValueError("native snapshot belongs to a different Helix store") + generation = snapshot.generation + meta = dict(snapshot.metadata) + else: + generation = self.active_generation + meta = self._metadata(generation) + topology_checksum = meta.get(_TOPOLOGY_CHECKSUM) + if not isinstance(topology_checksum, str): + raise RuntimeError("embedded Helix metadata has invalid state revision fields") + + current = previous_state if previous_state is not None else self.read_state() + for value in (current, state): + if not isinstance(value.get("build", {}), dict) or not isinstance( + value.get("incremental", {}), dict + ): + raise TypeError("durable build and incremental state must be mappings") + changed_kinds = [ + kind + for kind in _STATE_KINDS + if _state_category_value(current, kind, generation) + != _state_category_value(state, kind, generation) + ] + if not changed_kinds: + return + + revision = uuid.uuid4().hex + records_by_kind = { + kind: self._state_records_for_kind(state, generation, kind) + for kind in changed_kinds + } + changed_records = [ + record + for kind in changed_kinds + for record in records_by_kind[kind] + ] + category_checksums: dict[str, str] = {} + category_counts: dict[str, int] = {} + for kind in _STATE_KINDS: + if kind in records_by_kind: + category_checksums[kind] = _state_category_checksum( + records_by_kind[kind] + ) + category_counts[kind] = len(records_by_kind[kind]) + continue + checksum = meta.get(_STATE_CHECKSUM_KEYS[kind]) + count = meta.get(_STATE_COUNT_KEYS[kind]) + if not isinstance(checksum, str) or not isinstance(count, int): + raise RuntimeError( + "embedded Helix metadata has invalid state category fields" + ) + category_checksums[kind] = checksum + category_counts[kind] = count + state_checksum = _combined_state_checksum(category_checksums) + state_record_count = sum(category_counts.values()) + try: + traversal = self._helix.g().n_with_label_where( + _META_LABEL, + self._helix.SourcePredicate.eq(_GENERATION, generation), + ) + updates = { + "state_record_count": state_record_count, + "state_checksum": state_checksum, + "checksum": _generation_checksum(topology_checksum, state_checksum), + **{ + _STATE_REVISION_KEYS[kind]: revision + for kind in changed_kinds + }, + **{ + _STATE_CHECKSUM_KEYS[kind]: category_checksums[kind] + for kind in changed_kinds + }, + **{ + _STATE_COUNT_KEYS[kind]: category_counts[kind] + for kind in changed_kinds + }, + } + for key, value in updates.items(): + traversal = traversal.set_property(key, value) + transaction_size = len(changed_records) + len(changed_kinds) + 1 + atomic_write = transaction_size <= _STATE_WRITE_CHUNK_SIZE + batch = self._helix.write_batch() + if atomic_write: + for index, record in enumerate(changed_records): + batch = batch.var_as( + f"new_state_{index}", + self._helix.g().add_n( + _STATE_LABEL, + self._state_record_properties( + generation, revision, record + ), + ), + ) + else: + self._write_state_records(generation, changed_records, revision) + batch = batch.var_as("activate_state", traversal) + returned = ["activate_state"] + for index, kind in enumerate(changed_kinds): + old_revision = _state_revision(meta, kind) + if old_revision is None: + continue + variable = f"drop_old_state_{index}" + predicate = self._helix.SourcePredicate.and_(( + self._helix.SourcePredicate.eq(_GENERATION, generation), + self._helix.SourcePredicate.eq(_STATE_REVISION, old_revision), + self._helix.SourcePredicate.eq(_STATE_KIND, kind), + )) + batch = batch.var_as( + variable, + self._helix.g() + .n_with_label_where(_STATE_LABEL, predicate) + .drop(), + ) + returned.append(variable) + self._query( + batch.returning(returned) + ) + except Exception: + for kind in changed_kinds: + self._drop_state_revision(generation, revision, kind=kind) + raise + + @staticmethod + def _storage_key(generation: str, external_key: str) -> str: + return f"{generation}:{external_key}" + + @staticmethod + def _state_records( + state: dict[str, Any] | None, + ) -> list[tuple[str, str, Any, int]]: + if state is None: + return [] + records: list[tuple[str, str, Any, int]] = [] + order = 0 + for section, payload in state.items(): + if section == "communities" and isinstance(payload, list): + if not payload: + records.append(("section", section, [], order)) + order += 1 + for index, community in enumerate(payload): + records.append(("community", str(index), community, order)) + order += 1 + continue + if section == "incremental" and isinstance(payload, dict): + files = payload.get("files", {}) + if not isinstance(files, dict): + raise TypeError("incremental durable state files must be a mapping") + extraction_cache = payload.get("extraction_cache", {}) + if not isinstance(extraction_cache, dict): + raise TypeError( + "incremental durable state extraction cache must be a mapping" + ) + metadata = { + key: value + for key, value in payload.items() + if key not in {"files", "extraction_cache"} + } + if "files" in payload and not files: + metadata["files"] = {} + if "extraction_cache" in payload and not extraction_cache: + metadata["extraction_cache"] = {} + records.append(("section", section, metadata, order)) + order += 1 + for path, file_state in files.items(): + if not isinstance(path, str): + raise TypeError( + "incremental durable state file paths must be strings" + ) + records.append(("file", path, file_state, order)) + order += 1 + for cache_key, cache_value in extraction_cache.items(): + if not isinstance(cache_key, str): + raise TypeError( + "incremental durable extraction cache keys must be strings" + ) + records.append(("cache", cache_key, cache_value, order)) + order += 1 + continue + records.append(("section", section, payload, order)) + order += 1 + kind_order = {kind: 0 for kind in _STATE_KINDS} + normalized: list[tuple[str, str, Any, int]] = [] + for kind, key, payload, _ in records: + normalized.append((kind, key, payload, kind_order[kind])) + kind_order[kind] += 1 + kind_rank = {kind: index for index, kind in enumerate(_STATE_KINDS)} + normalized.sort(key=lambda item: (kind_rank[item[0]], item[3])) + return normalized + + def _state_records_for_kind( + self, state: dict[str, Any], generation: str, kind: str + ) -> list[tuple[str, str, Any, int]]: + value = _state_category_value(state, kind, generation) + if kind == "section": + partial = value + elif kind == "community": + partial = {"communities": value} + else: + partial = {"incremental": { + "files" if kind == "file" else "extraction_cache": value + }} + encoded = _json_value( + _encode_state_value(partial), f"durable {kind} state" + ) + return [ + record for record in self._state_records(encoded) + if record[0] == kind + ] + + def _stage_generation( + self, + generation: str, + manifest: dict[str, Any], + nodes: list[ + tuple[str, dict[str, Any], dict[str, Any], dict[str, str]] + ], + edges: list[ + tuple[str, str, str, dict[str, Any], str, dict[str, Any]] + ], + state: dict[str, Any] | None, + ) -> None: + metadata = {**manifest, _GENERATION: generation} + self._query( + self._helix.write_batch() + .var_as("meta", self._helix.g().add_n(_META_LABEL, metadata)) + .returning(["meta"]) + ) + + node_ids: dict[str, int] = {} + for offset in range(0, len(nodes), _WRITE_CHUNK_SIZE): + batch = self._helix.write_batch() + returned: list[str] = [] + encoded_ids: list[str] = [] + for local_index, (encoded_id, identity, attrs, search) in enumerate( + nodes[offset : offset + _WRITE_CHUNK_SIZE] + ): + index = offset + local_index + variable = f"node_{local_index}" + returned.append(variable) + encoded_ids.append(encoded_id) + batch = batch.var_as( + variable, + self._helix.g().add_n( + _NODE_LABEL, + { + _GENERATION: generation, + _STORAGE_KEY: self._storage_key(generation, encoded_id), + _EXTERNAL_KEY: identity, + _ATTRS: attrs, + _ORDER: index, + **search, + }, + ), + ) + result = self._query(batch.returning(returned)) + for encoded_id, variable in zip(encoded_ids, returned): + node_ids[encoded_id] = _returned_node_id(result, variable) + + if len(node_ids) != len(nodes): + raise RuntimeError( + "embedded Helix staged node count does not match the input graph" + ) + + for offset in range(0, len(edges), _STAGED_EDGE_WRITE_CHUNK_SIZE): + batch = self._helix.write_batch() + edge_returned = "" + for local_index, (source, target, key, identity, relation, attrs) in enumerate( + edges[offset : offset + _STAGED_EDGE_WRITE_CHUNK_SIZE] + ): + index = offset + local_index + edge_returned = f"edge_{local_index}" + batch = batch.var_as( + edge_returned, + self._helix.g() + .n(self._helix.NodeRef.ids([node_ids[source]])) + .add_e( + relation, + self._helix.NodeRef.ids([node_ids[target]]), + { + _GENERATION: generation, + _EDGE_KEY: identity, + _ATTRS: attrs, + _ORDER: index, + **( + {_EDGE_CONTEXT: attrs["context"]} + if isinstance(attrs.get("context"), str) + and attrs["context"] + else {} + ), + **( + {_NATIVE_WEIGHT: float(attrs["weight"])} + if isinstance(attrs.get("weight"), (int, float)) + and not isinstance(attrs.get("weight"), bool) + and math.isfinite(float(attrs["weight"])) + else {} + ), + }, + ), + ) + self._query(batch.returning([edge_returned])) + + self._write_state_records(generation, self._state_records(state), generation) + + def _write_state_records( + self, + generation: str, + state_records: list[tuple[str, str, Any, int]], + revision: str, + ) -> None: + # State rows are independent native records. A fixed planner-safe batch + # size keeps transaction cost bounded without exception-driven retries. + for offset in range(0, len(state_records), _STATE_WRITE_CHUNK_SIZE): + self._write_state_chunk( + generation, + state_records[offset : offset + _STATE_WRITE_CHUNK_SIZE], + revision, + ) + + def _write_state_chunk( + self, + generation: str, + records: list[tuple[str, str, Any, int]], + revision: str, + ) -> None: + batch = self._helix.write_batch() + returned = "" + for local_index, record in enumerate(records): + returned = f"state_{local_index}" + batch = batch.var_as( + returned, + self._helix.g().add_n( + _STATE_LABEL, + self._state_record_properties(generation, revision, record), + ), + ) + self._query(batch.returning([returned])) + + @staticmethod + def _state_record_properties( + generation: str, + revision: str, + record: tuple[str, str, Any, int], + ) -> dict[str, Any]: + kind, key, payload, order = record + properties = { + _GENERATION: generation, + _STATE_REVISION: revision, + _STATE_KIND: kind, + _STATE_KEY: key, + _STATE_PAYLOAD: "json:" + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ), + _ORDER: order, + } + if kind == "community" and isinstance(payload, dict): + for name in ( + "id", "name", "naming_source", "signature", "cohesion", + ): + if name in payload and payload[name] is not None: + properties[name] = payload[name] + elif kind == "file" and isinstance(payload, dict): + properties["relative_path"] = key + for name in ("content_hash", "semantic_hash"): + if name in payload: + properties[name] = payload[name] + return properties + + def _active_generation(self, *, required: bool = True) -> str | None: + batch = ( + self._helix.read_batch() + .var_as( + "control", + self._helix.g() + .n_with_label_where( + _CONTROL_LABEL, + self._helix.SourcePredicate.eq(_CONTROL_KEY, "active"), + ) + .value_map(), + ) + .returning(["control"]) + ) + rows = _rows(self._query(batch), "control") + if not rows: + if required: + raise RuntimeError("embedded Helix graph has no active generation") + return None + if len(rows) != 1: + raise RuntimeError("embedded Helix graph has duplicated control rows") + generation = _properties(rows[0], "control").get(_ACTIVE_GENERATION) + if not isinstance(generation, str): + raise RuntimeError("embedded Helix graph control row has no active generation") + return generation + + def _activate_generation( + self, + generation: str, + *, + create: bool, + previous: str | None, + retain_previous: bool, + ) -> None: + if create: + traversal = self._helix.g().add_n( + _CONTROL_LABEL, + {_CONTROL_KEY: "active", _ACTIVE_GENERATION: generation}, + ) + else: + traversal = self._helix.g().n_with_label_where( + _CONTROL_LABEL, + self._helix.SourcePredicate.eq(_CONTROL_KEY, "active"), + ).set_property(_ACTIVE_GENERATION, generation) + if retain_previous and previous is not None: + traversal = traversal.set_property(_PREVIOUS_GENERATION, previous) + else: + traversal = traversal.remove_property(_PREVIOUS_GENERATION) + batch = self._helix.write_batch().var_as("activate", traversal).returning(["activate"]) + self._query(batch) + + def rollback(self) -> LoadedGraph: + """Activate the explicitly retained previous generation.""" + if self._read_only: + raise RuntimeError("cannot roll back through a read-only embedded Helix store") + control = self._control_properties() + assert control is not None + active = control.get(_ACTIVE_GENERATION) + previous = control.get(_PREVIOUS_GENERATION) + if not isinstance(active, str) or not isinstance(previous, str): + raise RuntimeError( + "no rollback generation was retained; rebuild or update with --retain-rollback" + ) + self.load_generation(previous, attach_query=False) + self._activate_generation( + previous, + create=False, + previous=active, + retain_previous=True, + ) + self._cleanup_inactive_generations() + return self.load_generation(previous) + + def _drop_generation(self, generation: str) -> None: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + batch = ( + self._helix.write_batch() + .var_as( + "drop_edges", + self._helix.g().e_where(predicate).drop(), + ) + .var_as( + "drop_nodes", + self._helix.g().n_with_label_where(_NODE_LABEL, predicate).drop(), + ) + .var_as( + "drop_meta", + self._helix.g().n_with_label_where(_META_LABEL, predicate).drop(), + ) + .var_as( + "drop_state", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).drop(), + ) + .returning(["drop_edges", "drop_nodes", "drop_meta", "drop_state"]) + ) + self._query(batch) + + def _drop_state_revision( + self, generation: str, revision: str, *, kind: str | None = None + ) -> None: + predicates = [ + self._helix.SourcePredicate.eq(_GENERATION, generation), + self._helix.SourcePredicate.eq(_STATE_REVISION, revision), + ] + if kind is not None: + predicates.append(self._helix.SourcePredicate.eq(_STATE_KIND, kind)) + predicate = self._helix.SourcePredicate.and_(predicates) + self._query( + self._helix.write_batch() + .var_as( + "drop_state_revision", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).drop(), + ) + .returning(["drop_state_revision"]) + ) + + def _cleanup_inactive_generations(self) -> None: + active = self._active_generation(required=False) + retained = {active} if active is not None else set() + control = self._control_properties(required=False) + previous = control.get(_PREVIOUS_GENERATION) if control is not None else None + if isinstance(previous, str): + retained.add(previous) + batch = ( + self._helix.read_batch() + .var_as("meta", self._helix.g().n_with_label(_META_LABEL).value_map()) + .returning(["meta"]) + ) + generations = { + generation + for raw in _rows(self._query(batch), "meta") + if isinstance( + generation := _properties(raw, "metadata").get(_GENERATION), str + ) + } + for generation in generations: + if generation not in retained: + self._drop_generation(generation) + + def _cleanup_inactive_state_revisions(self) -> None: + batch = ( + self._helix.read_batch() + .var_as("meta", self._helix.g().n_with_label(_META_LABEL).value_map()) + .returning(["meta"]) + ) + for raw in _rows(self._query(batch), "meta"): + row = _properties(raw, "metadata") + generation = row.get(_GENERATION) + if not isinstance(generation, str): + continue + for kind in _STATE_KINDS: + revision = _state_revision(row, kind) + if revision is None: + continue + predicate = self._helix.SourcePredicate.and_(( + self._helix.SourcePredicate.eq(_GENERATION, generation), + self._helix.SourcePredicate.eq(_STATE_KIND, kind), + self._helix.SourcePredicate.neq(_STATE_REVISION, revision), + )) + self._query( + self._helix.write_batch() + .var_as( + "drop_inactive_state", + self._helix.g() + .n_with_label_where(_STATE_LABEL, predicate) + .drop(), + ) + .returning(["drop_inactive_state"]) + ) + + def _control_properties(self, *, required: bool = True) -> dict[str, Any] | None: + batch = ( + self._helix.read_batch() + .var_as( + "control", + self._helix.g() + .n_with_label_where( + _CONTROL_LABEL, + self._helix.SourcePredicate.eq(_CONTROL_KEY, "active"), + ) + .value_map(), + ) + .returning(["control"]) + ) + rows = _rows(self._query(batch), "control") + if not rows: + if required: + raise RuntimeError("embedded Helix graph has no active generation") + return None + if len(rows) != 1: + raise RuntimeError("embedded Helix graph has duplicated control rows") + return _properties(rows[0], "control") + + def _read_rows( + self, generation: str + ) -> tuple[dict[str, Any], list[Any], list[Any], list[Any]]: + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + batch = ( + self._helix.read_batch() + .var_as( + "meta", + self._helix.g().n_with_label_where(_META_LABEL, predicate).value_map(), + ) + .var_as( + "nodes", + self._helix.g().n_with_label_where(_NODE_LABEL, predicate).value_map(), + ) + .var_as( + "edges", + self._helix.g().e_where(predicate).value_map(), + ) + .var_as( + "state", + self._helix.g().n_with_label_where(_STATE_LABEL, predicate).value_map(), + ) + .returning(["meta", "nodes", "edges", "state"]) + ) + result = self._query(batch) + meta_rows = _rows(result, "meta") + if len(meta_rows) != 1: + raise RuntimeError( + f"embedded Helix graph must contain exactly one metadata node; " + f"found {len(meta_rows)}" + ) + return ( + _properties(meta_rows[0], "metadata"), + _rows(result, "nodes"), + _rows(result, "edges"), + _rows(result, "state"), + ) + + def _read_state_rows( + self, + generation: str, + *, + metadata: dict[str, Any] | None = None, + revision: str | None = None, + ) -> list[Any]: + generation_predicate = self._helix.SourcePredicate.eq( + _GENERATION, generation + ) + if revision is not None: + predicate = self._helix.SourcePredicate.and_(( + generation_predicate, + self._helix.SourcePredicate.eq(_STATE_REVISION, revision), + )) + else: + meta = metadata or self._metadata(generation) + kind_predicates = [] + for kind in _STATE_KINDS: + selected_revision = _state_revision(meta, kind) + if selected_revision is None: + predicate = generation_predicate + break + kind_predicates.append(self._helix.SourcePredicate.and_(( + self._helix.SourcePredicate.eq(_STATE_KIND, kind), + self._helix.SourcePredicate.eq( + _STATE_REVISION, selected_revision + ), + ))) + else: + predicate = self._helix.SourcePredicate.and_(( + generation_predicate, + self._helix.SourcePredicate.or_(kind_predicates), + )) + batch = ( + self._helix.read_batch() + .var_as( + "state", + self._helix.g() + .n_with_label_where( + _STATE_LABEL, + predicate, + ) + .value_map(), + ) + .returning(["state"]) + ) + return _rows(self._query(batch), "state") + + @staticmethod + def _state_records_from_rows( + rows: list[Any], + ) -> list[tuple[str, str, Any, int]]: + ordered: list[tuple[str, str, Any, int]] = [] + seen_orders: set[tuple[str, int]] = set() + for raw in rows: + row = _properties(raw, "durable state") + kind = row.get(_STATE_KIND) + key = row.get(_STATE_KEY) + order = row.get(_ORDER) + if ( + kind not in {"section", "community", "file", "cache"} + or not isinstance(key, str) + or not isinstance(order, int) + or isinstance(order, bool) + or _STATE_PAYLOAD not in row + ): + raise RuntimeError( + "embedded Helix durable state record is missing schema fields" + ) + order_key = (kind, order) + if order_key in seen_orders: + raise RuntimeError("embedded Helix durable state has duplicate ordering") + seen_orders.add(order_key) + payload = row[_STATE_PAYLOAD] + if isinstance(payload, str) and payload.startswith("json:"): + try: + payload = json.loads(payload[5:]) + except json.JSONDecodeError as exc: + raise RuntimeError( + "embedded Helix durable state has invalid encoded payload" + ) from exc + ordered.append((kind, key, payload, order)) + kind_rank = {kind: index for index, kind in enumerate(_STATE_KINDS)} + ordered.sort(key=lambda item: (kind_rank[item[0]], item[3])) + return ordered + + @staticmethod + def _state_from_rows(rows: list[Any], expected_count: Any) -> dict[str, Any]: + if not isinstance(expected_count, int) or isinstance(expected_count, bool): + raise RuntimeError("embedded Helix metadata has an invalid state record count") + if len(rows) != expected_count: + raise RuntimeError( + "embedded Helix durable state failed count verification: " + f"expected {expected_count}, read {len(rows)}" + ) + + ordered = HelixEmbeddedStore._state_records_from_rows(rows) + + state: dict[str, Any] = {} + community_records: list[Any] = [] + file_records: dict[str, Any] = {} + cache_records: dict[str, Any] = {} + for kind, key, payload, _ in ordered: + if kind == "section": + if key in state: + raise RuntimeError( + f"embedded Helix durable state duplicates section {key!r}" + ) + state[key] = payload + elif kind == "community": + community_records.append(payload) + elif kind == "file": + if key in file_records: + raise RuntimeError( + f"embedded Helix durable state duplicates file {key!r}" + ) + file_records[key] = payload + else: + if key in cache_records: + raise RuntimeError( + f"embedded Helix durable state duplicates cache key {key!r}" + ) + cache_records[key] = payload + + if community_records: + if "communities" in state: + raise RuntimeError( + "embedded Helix durable state mixes community section and records" + ) + state["communities"] = community_records + if file_records: + incremental = state.setdefault("incremental", {}) + if not isinstance(incremental, dict) or "files" in incremental: + raise RuntimeError( + "embedded Helix durable state has invalid incremental records" + ) + incremental["files"] = file_records + if cache_records: + incremental = state.setdefault("incremental", {}) + if not isinstance(incremental, dict) or "extraction_cache" in incremental: + raise RuntimeError( + "embedded Helix durable state has invalid extraction cache records" + ) + incremental["extraction_cache"] = cache_records + return state + + @staticmethod + def _verified_state_from_rows( + rows: list[Any], metadata: dict[str, Any] + ) -> dict[str, Any]: + state = HelixEmbeddedStore._state_from_rows( + rows, metadata.get("state_record_count") + ) + has_category_checksums = all( + isinstance(metadata.get(_STATE_CHECKSUM_KEYS[kind]), str) + and isinstance(metadata.get(_STATE_COUNT_KEYS[kind]), int) + for kind in _STATE_KINDS + ) + if has_category_checksums: + records = HelixEmbeddedStore._state_records_from_rows(rows) + checksums: dict[str, str] = {} + for kind in _STATE_KINDS: + category = [record for record in records if record[0] == kind] + expected_count = metadata[_STATE_COUNT_KEYS[kind]] + if len(category) != expected_count: + raise RuntimeError( + "embedded Helix durable state category failed count verification" + ) + checksum = _state_category_checksum(category) + if checksum != metadata[_STATE_CHECKSUM_KEYS[kind]]: + raise RuntimeError( + "embedded Helix durable state category failed checksum verification" + ) + checksums[kind] = checksum + actual_checksum = _combined_state_checksum(checksums) + else: + actual_checksum = _checksum(state) + if metadata.get("state_checksum") != actual_checksum: + raise RuntimeError("embedded Helix durable state failed checksum verification") + return state + + def read_data(self) -> dict[str, Any]: + generation = self._active_generation() + assert generation is not None + return self._read_generation_data(generation) + + @property + def active_generation(self) -> str: + generation = self._active_generation() + assert generation is not None + return generation + + def read_state(self) -> dict[str, Any]: + generation = self.active_generation + meta = self._metadata(generation) + self._validate_metadata(meta) + state = self._verified_state_from_rows( + self._read_state_rows(generation, metadata=meta), meta + ) + decoded = _decode_state_value(state) + if not isinstance(decoded, dict): + raise RuntimeError("embedded Helix generation contains invalid durable state") + return decoded + + def native_graph( + self, + generation: str | None = None, + *, + metadata: dict[str, Any] | None = None, + ) -> Any: + """Load one immutable native snapshot of the active generation.""" + generation = generation or self.active_generation + meta = metadata or self._metadata(generation) + node_count = meta.get("node_count") + edge_count = meta.get("edge_count") + if ( + not isinstance(node_count, int) + or not isinstance(edge_count, int) + or node_count > self._max_nodes + or edge_count > self._max_edges + ): + raise RuntimeError( + "embedded Helix generation exceeds configured snapshot bounds" + ) + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + kind = ( + "multidigraph" if bool(meta.get("directed")) and bool(meta.get("multigraph")) + else "digraph" if bool(meta.get("directed")) + else "multigraph" if bool(meta.get("multigraph")) + else "graph" + ) + selection = self._helix.GraphSelection( + node_traversal=self._helix.g().n_with_label_where(_NODE_LABEL, predicate), + edge_traversal=self._helix.g().e_where(predicate), + kind=kind, + metadata=self._helix.GraphMetadataSelection( + self._helix.g().n_with_label_where(_META_LABEL, predicate), + ( + _GENERATION, + "schema_version", + "directed", + "multigraph", + "graph", + "extras", + "node_count", + "edge_count", + _ACTIVE_STATE_REVISION, + *_STATE_REVISION_KEYS.values(), + *_STATE_CHECKSUM_KEYS.values(), + *_STATE_COUNT_KEYS.values(), + _CHECKSUM_MODE, + _TOPOLOGY_CHECKSUM, + "state_checksum", + "checksum", + ), + ), + node_identity=self._helix.IdentitySelection.tagged_property(_EXTERNAL_KEY), + graphify_edge_key=( + self._helix.IdentitySelection.tagged_property(_EDGE_KEY) + if bool(meta.get("multigraph")) + else None + ), + weight_property=_NATIVE_WEIGHT, + node_properties=(_ATTRS, _ORDER), + edge_properties=(_ATTRS, _ORDER), + max_nodes=self._max_nodes, + max_edges=self._max_edges, + allow_full_scan=True, + ) + graph = self._client.graph(selection) + if graph.node_count != meta.get("node_count") or graph.edge_count != meta.get("edge_count"): + raise RuntimeError("native Helix snapshot failed generation count verification") + return graph + + def _read_generation_data(self, generation: str) -> dict[str, Any]: + meta = self._metadata(generation) + self._validate_metadata(meta) + native = self.native_graph(generation, metadata=meta) + + nodes_with_order: list[tuple[int, dict[str, Any]]] = [] + for record in native.nodes(): + projected = dict(record.attributes) + attrs, order = projected.get(_ATTRS), projected.get(_ORDER) + if not isinstance(attrs, dict) or not isinstance(order, int): + raise RuntimeError("native Helix node is missing Graphify schema fields") + nodes_with_order.append((order, {"id": record.id, **attrs})) + + edges_with_order: list[tuple[int, dict[str, Any]]] = [] + multigraph = bool(meta.get("multigraph", False)) + for record in native.edges(): + projected = dict(record.attributes) + attrs, order = projected.get(_ATTRS), projected.get(_ORDER) + if not isinstance(attrs, dict) or not isinstance(order, int): + raise RuntimeError("native Helix edge is missing Graphify schema fields") + edge = { + "source": record.source, + "target": record.target, + "relation": record.label, + **attrs, + } + if multigraph: + edge["key"] = record.graphify_key + edges_with_order.append((order, edge)) + + nodes_with_order.sort(key=lambda item: item[0]) + edges_with_order.sort(key=lambda item: item[0]) + extras = meta.get("extras", {}) + graph_attrs = meta.get("graph", {}) + if not isinstance(extras, dict) or not isinstance(graph_attrs, dict): + raise RuntimeError("embedded Helix metadata contains invalid graph attributes") + topology_payload = { + "directed": bool(meta.get("directed", False)), + "multigraph": multigraph, + "graph": graph_attrs, + "nodes": [row for _, row in nodes_with_order], + "links": [row for _, row in edges_with_order], + **extras, + } + payload = dict(topology_payload) + state = self._verified_state_from_rows( + self._read_state_rows(generation, metadata=meta), meta + ) + if state: + payload[_DURABLE_STATE] = state + expected_nodes = meta.get("node_count") + expected_edges = meta.get("edge_count") + expected_checksum = meta.get("checksum") + if expected_nodes != len(nodes_with_order) or expected_edges != len(edges_with_order): + raise RuntimeError( + "embedded Helix graph failed count verification: " + f"expected {expected_nodes}/{expected_edges}, " + f"read {len(nodes_with_order)}/{len(edges_with_order)}" + ) + if meta.get(_CHECKSUM_MODE) in { + _SPLIT_CHECKSUM_MODE, _STREAM_CHECKSUM_MODE, + }: + if meta.get(_CHECKSUM_MODE) == _STREAM_CHECKSUM_MODE: + stream_checksum = _TopologyStreamChecksum( + directed=topology_payload["directed"], + multigraph=topology_payload["multigraph"], + graph=graph_attrs, + extras=extras, + ) + for node in topology_payload["nodes"]: + stream_checksum.node(node) + for edge in topology_payload["links"]: + stream_checksum.edge(edge) + topology_checksum = stream_checksum.hexdigest() + else: + topology_checksum = _checksum(topology_payload) + expected_topology_checksum = meta.get(_TOPOLOGY_CHECKSUM) + if expected_topology_checksum != topology_checksum: + raise RuntimeError( + "embedded Helix topology failed checksum verification: " + f"expected {expected_topology_checksum!r}, got {topology_checksum!r}" + ) + state_checksum = meta.get("state_checksum") + if not isinstance(state_checksum, str): + raise RuntimeError( + "embedded Helix metadata has no durable state checksum" + ) + actual_checksum = _generation_checksum(topology_checksum, state_checksum) + else: + actual_checksum = _checksum(payload) + if expected_checksum != actual_checksum: + raise RuntimeError( + "embedded Helix graph failed checksum verification: " + f"expected {expected_checksum!r}, got {actual_checksum!r}" + ) + return payload + + def _verify_generation_counts(self, generation: str) -> dict[str, Any]: + """Verify one staged generation with bounded native count projections.""" + metadata = self._metadata(generation) + self._validate_metadata(metadata) + predicate = self._helix.SourcePredicate.eq(_GENERATION, generation) + counts = self._query( + self._helix.read_batch() + .var_as( + "nodes", + self._helix.g() + .n_with_label_where(_NODE_LABEL, predicate) + .count(), + ) + .var_as("edges", self._helix.g().e_where(predicate).count()) + .var_as( + "state_records", + self._helix.g() + .n_with_label_where(_STATE_LABEL, predicate) + .count(), + ) + .returning(["nodes", "edges", "state_records"]) + ) + if not isinstance(counts, dict): + raise RuntimeError("embedded Helix count verification returned no result") + expected = { + "nodes": metadata.get("node_count"), + "edges": metadata.get("edge_count"), + "state_records": metadata.get("state_record_count"), + } + if any( + not isinstance(expected[name], int) or counts.get(name) != expected[name] + for name in expected + ): + raise RuntimeError( + "embedded Helix generation failed count verification: " + f"expected {expected!r}, read {counts!r}" + ) + return { + "schema_version": _SCHEMA_VERSION, + **counts, + "checksum": metadata.get("checksum"), + } + + @staticmethod + def _validate_metadata(meta: dict[str, Any]) -> None: + if meta.get("schema_version") != _SCHEMA_VERSION: + raise RuntimeError( + "unsupported embedded Helix graph schema: " + f"expected {_SCHEMA_VERSION}, got {meta.get('schema_version')!r}" + ) + + def load_generation( + self, generation: str, *, attach_query: bool = True + ) -> LoadedGraph: + meta = self._metadata(generation) + self._validate_metadata(meta) + native = self.native_graph(generation, metadata=meta) + state = self._verified_state_from_rows( + self._read_state_rows(generation, metadata=meta), meta + ) + decoded = _decode_state_value(state) + if not isinstance(decoded, dict): + raise RuntimeError("embedded Helix generation contains invalid durable state") + query = HelixNodeQuery(self.path, generation) if attach_query else None + return LoadedGraph(native, generation, decoded, meta, self.path, query) + + def load(self) -> LoadedGraph: + return self.load_generation(self.active_generation) + + def checkpoint(self) -> None: + # Embedded write queries await the storage commit; close() flushes the handle. + if self._closed: + raise RuntimeError("embedded Helix store is closed") + + def _metadata(self, generation: str) -> dict[str, Any]: + batch = ( + self._helix.read_batch() + .var_as( + "meta", + self._helix.g() + .n_with_label_where( + _META_LABEL, + self._helix.SourcePredicate.eq(_GENERATION, generation), + ) + .value_map(), + ) + .returning(["meta"]) + ) + rows = _rows(self._query(batch), "meta") + if len(rows) != 1: + raise RuntimeError("embedded Helix graph metadata is missing or duplicated") + return _properties(rows[0], "metadata") + + def verify(self) -> dict[str, Any]: + """Perform a deep topology/state checksum verification.""" + payload = self.read_data() + metadata = self._metadata(self.active_generation) + return { + "schema_version": _SCHEMA_VERSION, + "nodes": len(payload["nodes"]), + "edges": len(payload["links"]), + "checksum": metadata.get("checksum"), + } + + def verify_counts(self) -> dict[str, Any]: + """Perform the lightweight verification used on the write path.""" + return self._verify_generation_counts(self.active_generation) + + def close(self) -> None: + if not self._closed: + try: + _close_public_client(self._client) + finally: + self._store_lock.release() + self._closed = True + + def __enter__(self) -> "HelixEmbeddedStore": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + def __del__(self) -> None: + try: + if hasattr(self, "_closed") and not self._closed: + self.close() + except Exception: + pass + + +class HelixGraphReader: + """Retain one native graph while polling immutable embedded snapshots.""" + + def __init__(self, path: str | Path = DEFAULT_PROJECT_STORE) -> None: + self.path = Path(path) + self._version: tuple[str | None, ...] | None = None + self._graph: LoadedGraph | None = None + self._lock = threading.RLock() + + def get(self) -> LoadedGraph: + with self._lock: + # Helix embedded readers are immutable database snapshots. Reopen + # the lightweight handle to observe a writer's atomic pointer flip; + # retain the expensive native graph while its version is unchanged. + with HelixEmbeddedStore(self.path, read_only=True) as store: + generation = store.active_generation + metadata = store._metadata(generation) + version = ( + generation, + *( + _state_revision(metadata, kind) + for kind in _STATE_KINDS + ), + ) + if self._graph is None or version != self._version: + if self._graph is not None and self._graph.query is not None: + self._graph.query.close() + self._graph = store.load_generation(generation) + self._version = version + return self._graph + + def close(self) -> None: + with self._lock: + if self._graph is not None and self._graph.query is not None: + self._graph.query.close() + self._graph = None + self._version = None + + def __del__(self) -> None: + try: + self.close() + except Exception: + pass + + +def persist_graph( + graph: GraphBuildData, + path: str | Path = DEFAULT_PROJECT_STORE, + *, + state: dict[str, Any] | None = None, + retain_rollback: bool = False, +) -> None: + with HelixEmbeddedStore(path, retain_rollback=retain_rollback) as store: + store.save(graph, state=state) + + +def persist_graph_data( + data: dict[str, Any], + path: str | Path, + *, + retain_rollback: bool = False, +) -> None: + with HelixEmbeddedStore(path, retain_rollback=retain_rollback) as store: + store.save_data(data) + + +def load_graph(path: str | Path = DEFAULT_PROJECT_STORE) -> LoadedGraph: + with HelixEmbeddedStore(path, read_only=True) as store: + return store.load() + + +def graph_storage_exists(path: str | Path = DEFAULT_PROJECT_STORE) -> bool: + return Path(path).expanduser().is_dir() + + +__all__ = [ + "HelixEmbeddedStore", + "HelixGraphReader", + "HelixNodeQuery", + "DEFAULT_GLOBAL_STORE", + "DEFAULT_MAX_EDGES", + "DEFAULT_MAX_NODES", + "DEFAULT_PROJECT_STORE", + "graph_storage_exists", + "load_graph", + "persist_graph", + "persist_graph_data", +] diff --git a/graphify/helix/state.py b/graphify/helix/state.py new file mode 100644 index 000000000..c2a17047c --- /dev/null +++ b/graphify/helix/state.py @@ -0,0 +1,103 @@ +"""Versioned durable state stored inside the same Helix graph generation.""" + +from __future__ import annotations + +import hashlib +from typing import Any + + +STATE_SCHEMA_VERSION = 1 + + +def new_state(**overrides: Any) -> dict[str, Any]: + state: dict[str, Any] = { + "schema_version": STATE_SCHEMA_VERSION, + "build": {}, + "communities": [], + "analysis": {}, + "incremental": { + "files": {}, + "extractor_state": {}, + "topology_sources": [], + }, + "learning": {}, + "semantic": {"used": False}, + } + state.update(overrides) + return state + + +def community_records( + communities: dict[int, list[Any]], + *, + labels: dict[int, str] | None = None, + cohesion: dict[int, float] | None = None, + naming_source: str = "generated", +) -> list[dict[str, Any]]: + labels = labels or {} + cohesion = cohesion or {} + from .persistence import _encode_key + + return [ + { + "id": int(cid), + "members": list(members), + "name": labels.get(cid, f"Community {cid}"), + "naming_source": naming_source, + "signature": "sha256:" + hashlib.sha256( + "\n".join(sorted(_encode_key(member) for member in members)).encode("utf-8") + ).hexdigest(), + "cohesion": cohesion.get(cid), + "clustering": { + "algorithm": "helix-leiden", + "seed": 42, + "resolution": 1.0, + "randomness": 0.001, + "trials": 1, + "max_iterations": 100, + }, + } + for cid, members in sorted(communities.items()) + ] + + +def communities_from_state(state: dict[str, Any]) -> dict[int, list[Any]]: + result: dict[int, list[Any]] = {} + for record in state.get("communities", []): + if isinstance(record, dict) and isinstance(record.get("id"), int): + result[record["id"]] = list(record.get("members", [])) + return result + + +def labels_from_state(state: dict[str, Any]) -> dict[int, str]: + return { + record["id"]: record["name"] + for record in state.get("communities", []) + if isinstance(record, dict) + and isinstance(record.get("id"), int) + and isinstance(record.get("name"), str) + } + + +def community_summaries( + graph: Any, + communities: dict[int, list[Any]], + labels: dict[int, str], +) -> list[dict[str, Any]]: + summaries = [] + for community_id, members in sorted(communities.items()): + member_set = set(members) + internal_edges = sum( + 1 + for edge in graph.edges() + if edge.source in member_set and edge.target in member_set + ) + summaries.append( + { + "id": community_id, + "name": labels.get(community_id, f"Community {community_id}"), + "node_count": len(members), + "internal_edge_count": internal_edges, + } + ) + return summaries diff --git a/graphify/hooks.py b/graphify/hooks.py index 77ef39cd3..28270abee 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -1,690 +1,129 @@ -# git hook integration - install/uninstall graphify post-commit and post-checkout hooks +"""Git hooks that keep an embedded Helix generation current.""" + from __future__ import annotations -import os + import re +import shlex +import subprocess import sys from pathlib import Path -_HOOK_MARKER = "# graphify-hook-start" -_HOOK_MARKER_END = "# graphify-hook-end" -_CHECKOUT_MARKER = "# graphify-checkout-hook-start" -_CHECKOUT_MARKER_END = "# graphify-checkout-hook-end" - -# __PINNED_PYTHON__ is replaced at install time with the absolute path of the -# Python interpreter that ran `graphify hook install`. For uv-tool and pipx -# installs the interpreter lives inside an isolated venv, so the launcher on -# PATH is the only entry point — and GUI git clients / CI runners often have a -# minimal PATH that omits ~/.local/bin. Pinning sys.executable at install time -# makes the hook work regardless of PATH at git-trigger time. -_PYTHON_DETECT = """\ -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs). -# _PINNED was recorded at hook-install time; tried first so the hook works even -# when the graphify launcher is not on PATH (common in GUI clients and CI). -# -# Probes check availability with importlib.util.find_spec instead of importing -# the package: a probe that imports graphify wholesale executes the full package -# import (10s+ cold on machines with AV-scanned or large site-packages) and used -# to run up to FOUR times synchronously, stalling every commit before the -# detached launch even started. find_spec locates the package without executing -# it, so each probe costs interpreter startup only. The detached rebuild still -# fails loudly in the log if the package is broken under that interpreter. -_GFY_PROBE="import importlib.util, sys; sys.exit(0 if importlib.util.find_spec('graphify') else 1)" -GRAPHIFY_PYTHON="" -_PINNED='__PINNED_PYTHON__' -if [ -n "$_PINNED" ] && [ -x "$_PINNED" ] && "$_PINNED" -c "$_GFY_PROBE" 2>/dev/null; then - GRAPHIFY_PYTHON="$_PINNED" -fi -# Second probe: read graphify-out/.graphify_python (written by the skill and -# CLI; survives uv-tool reinstalls and is the same source the README documents). -if [ -z "$GRAPHIFY_PYTHON" ]; then - _GFY_PYTHON_FILE="graphify-out/.graphify_python" - if [ -f "$_GFY_PYTHON_FILE" ]; then - _FROM_FILE=$(cat "$_GFY_PYTHON_FILE" 2>/dev/null | tr -d '[:space:]') - case "$_FROM_FILE" in - *[!a-zA-Z0-9/_.@:\\-]*) _FROM_FILE="" ;; # allowlist (covers Windows paths) - esac - if [ -n "$_FROM_FILE" ] && [ -x "$_FROM_FILE" ] && "$_FROM_FILE" -c "$_GFY_PROBE" 2>/dev/null; then - GRAPHIFY_PYTHON="$_FROM_FILE" - fi - fi -fi -# Third probe: resolve via the graphify launcher on PATH. -if [ -z "$GRAPHIFY_PYTHON" ]; then - GRAPHIFY_BIN=$(command -v graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - # Windows pip layout: Scripts/graphify(.exe) sits beside ..\\python.exe - # (or .\\python.exe inside a venv's Scripts dir). NOTE: command -v may - # return the launcher path WITHOUT the .exe suffix, so this cannot key - # on the extension. - _GFY_BINDIR=$(dirname "$GRAPHIFY_BIN") - if [ -x "$_GFY_BINDIR/../python.exe" ] && "$_GFY_BINDIR/../python.exe" -c "$_GFY_PROBE" 2>/dev/null; then - GRAPHIFY_PYTHON="$_GFY_BINDIR/../python.exe" - elif [ -x "$_GFY_BINDIR/python.exe" ] && "$_GFY_BINDIR/python.exe" -c "$_GFY_PROBE" 2>/dev/null; then - GRAPHIFY_PYTHON="$_GFY_BINDIR/python.exe" - fi - fi - if [ -z "$GRAPHIFY_PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - # POSIX launcher: parse the shebang. head -c + tr strip NUL bytes first — - # when the launcher is a Windows binary reached without its .exe suffix, - # a raw `head -1` reads binary into the command substitution and the - # shell warns about ignored null bytes on every commit. - case "$GRAPHIFY_BIN" in - *.exe) _SHEBANG="" ;; - *) _SHEBANG=$(head -c 256 "$GRAPHIFY_BIN" 2>/dev/null | tr -d '\\000' | head -n 1 | sed 's/^#![[:space:]]*//') ;; - esac - case "$_SHEBANG" in - */env\\ *) GRAPHIFY_PYTHON="${_SHEBANG#*/env }" ;; - *) GRAPHIFY_PYTHON="$_SHEBANG" ;; - esac - # Allowlist: only keep characters valid in a filesystem path to prevent - # injection if the shebang contains shell metacharacters. - case "$GRAPHIFY_PYTHON" in - *[!a-zA-Z0-9/_.@-]*) GRAPHIFY_PYTHON="" ;; - esac - if [ -n "$GRAPHIFY_PYTHON" ] && ! "$GRAPHIFY_PYTHON" -c "$_GFY_PROBE" 2>/dev/null; then - GRAPHIFY_PYTHON="" - fi - fi -fi -# Last resort: try python3 / python (works for system/venv installs on PATH). -if [ -z "$GRAPHIFY_PYTHON" ]; then - if command -v python3 >/dev/null 2>&1 && python3 -c "$_GFY_PROBE" 2>/dev/null; then - GRAPHIFY_PYTHON="python3" - elif command -v python >/dev/null 2>&1 && python -c "$_GFY_PROBE" 2>/dev/null; then - GRAPHIFY_PYTHON="python" - else - echo "[graphify hook] could not locate a Python with graphify installed. Add the graphify bin dir to PATH or re-run 'graphify hook install' from the env where graphify lives." >&2 - exit 0 - fi -fi -""" - -# The Python that the rebuild runs, shared by both hooks. Embedded verbatim into -# the launcher below and re-executed in the detached child. Must not contain the -# double-quote, $, backtick or backslash characters: it is carried inside a -# shell double-quoted `-c "..."` argument (see _detached_launch). -_REBUILD_BODY_COMMIT = """\ -import os, signal, sys -from pathlib import Path - -changed_raw = os.environ.get('GRAPHIFY_CHANGED', '') -changed = [Path(f.strip()) for f in changed_raw.strip().splitlines() if f.strip()] - -if not changed: - sys.exit(0) - -print(f'[graphify hook] {len(changed)} file(s) changed - rebuilding graph...') - -try: - from graphify.watch import _rebuild_code, _apply_resource_limits - _apply_resource_limits() - _timeout = int(os.environ.get('GRAPHIFY_REBUILD_TIMEOUT', '600')) - if _timeout > 0 and hasattr(signal, 'SIGALRM'): - signal.signal(signal.SIGALRM, lambda *_: (_ for _ in ()).throw(TimeoutError(f'graphify rebuild exceeded {_timeout}s'))) - signal.alarm(_timeout) - _force = os.environ.get('GRAPHIFY_FORCE', '').lower() in ('1', 'true', 'yes') - _root = Path('.') - _out = os.environ.get('GRAPHIFY_OUT', 'graphify-out') - _saved = Path(_out) / '.graphify_root' - if _saved.exists(): - _txt = _saved.read_text(encoding='utf-8').strip() - if _txt: - _root = Path(_txt) - _rebuild_code(_root, changed_paths=changed, force=_force) - # Refresh the work-memory lessons doc when saved Q&A outcomes exist - # (best-effort; never fails the hook). - try: - _md = (_root / _out) / 'memory' - if _md.is_dir() and any(_md.glob('*.md')): - from graphify.reflect import reflect as _reflect - _gj = (_root / _out) / 'graph.json' - _reflect(memory_dir=_md, out_path=(_root / _out) / 'reflections' / 'LESSONS.md', - graph_path=_gj if _gj.exists() else None) - except Exception: - pass -except TimeoutError as exc: - print(f'[graphify hook] {exc}') - sys.exit(1) -except Exception as exc: - print(f'[graphify hook] Rebuild failed: {exc}') - sys.exit(1) -""" - -_REBUILD_BODY_CHECKOUT = """\ -from graphify.watch import _rebuild_code, _apply_resource_limits -from pathlib import Path -import os, signal, sys -try: - _apply_resource_limits() - _timeout = int(os.environ.get('GRAPHIFY_REBUILD_TIMEOUT', '600')) - if _timeout > 0 and hasattr(signal, 'SIGALRM'): - signal.signal(signal.SIGALRM, lambda *_: (_ for _ in ()).throw(TimeoutError(f'graphify rebuild exceeded {_timeout}s'))) - signal.alarm(_timeout) - _force = os.environ.get('GRAPHIFY_FORCE', '').lower() in ('1', 'true', 'yes') - # post-checkout: branch switch can touch arbitrary files; full rebuild path - # (no changed_paths) is correct here. The flock inside _rebuild_code still - # prevents pile-ups when commit + checkout fire back-to-back. - _root = Path('.') - _out = os.environ.get('GRAPHIFY_OUT', 'graphify-out') - _saved = Path(_out) / '.graphify_root' - if _saved.exists(): - _txt = _saved.read_text(encoding='utf-8').strip() - if _txt: - _root = Path(_txt) - _rebuild_code(_root, force=_force) - # Refresh the work-memory lessons doc when saved Q&A outcomes exist - # (best-effort; never fails the hook). - try: - _md = (_root / _out) / 'memory' - if _md.is_dir() and any(_md.glob('*.md')): - from graphify.reflect import reflect as _reflect - _gj = (_root / _out) / 'graph.json' - _reflect(memory_dir=_md, out_path=(_root / _out) / 'reflections' / 'LESSONS.md', - graph_path=_gj if _gj.exists() else None) - except Exception: - pass -except TimeoutError as exc: - print(f'[graphify] {exc}') - sys.exit(1) -except Exception as exc: - print(f'[graphify] Rebuild failed: {exc}') - sys.exit(1) -""" - -# Cross-platform detached-launch shim (#1161). The hooks used to background the -# rebuild with `nohup "$GRAPHIFY_PYTHON" -c "..." &`, but Git for Windows' bundled -# MSYS shell ships no nohup (nor setsid), so that line died with -# 'nohup: command not found' and the rebuild silently never ran — git commit/pull -# still returned 0, so the graph just went stale with no signal. graphify already -# requires Python, so we let Python do the detaching: a tiny outer process spawns -# the real rebuild fully detached and returns immediately, so the hook never -# blocks. POSIX uses start_new_session (the setsid equivalent); Windows uses -# DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP, breaking away from any job object -# when allowed. This payload is carried inside a shell double-quoted -c argument, -# so it deliberately uses only single-quoted Python strings (no ", $, ` or \\). -_LAUNCHER_TEMPLATE = """\ -import os, subprocess, sys -_src = ''' -__REBUILD_BODY__ -''' -_log = os.environ.get('GRAPHIFY_REBUILD_LOG') or os.path.join(os.path.expanduser('~'), '.cache', 'graphify-rebuild.log') -try: - os.makedirs(os.path.dirname(_log), exist_ok=True) - _out = open(_log, 'a', buffering=1, encoding='utf-8', errors='replace') -except OSError: - _out = subprocess.DEVNULL -_kw = dict(stdout=_out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, cwd=os.getcwd(), close_fds=True) -_cmd = [sys.executable, '-c', _src] -if os.name == 'nt': - _flags = 0x00000008 | 0x00000200 # DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP - try: - subprocess.Popen(_cmd, creationflags=_flags | 0x01000000, **_kw) # + CREATE_BREAKAWAY_FROM_JOB - except OSError: - subprocess.Popen(_cmd, creationflags=_flags, **_kw) -else: - subprocess.Popen(_cmd, start_new_session=True, **_kw) -""" - - -def _detached_launch(rebuild_body: str) -> str: - """Return a POSIX-sh line that runs ``rebuild_body`` as a detached background - Python process via ``$GRAPHIFY_PYTHON``. - - Replaces the old ``nohup ... &`` form, which failed on Git for Windows' - shell (no nohup/setsid) and let the rebuild silently never run (#1161). - The launcher writes the child's output to ``$GRAPHIFY_REBUILD_LOG`` and - returns the instant the child is spawned, so the git hook never blocks. - """ - launcher = _LAUNCHER_TEMPLATE.replace("__REBUILD_BODY__", rebuild_body) - return '"$GRAPHIFY_PYTHON" -c "' + launcher + '"\n' - - -# Skip the rebuild inside a linked worktree (git worktree add), shared by both -# hooks. With core.hooksPath shared across worktrees a commit in any worktree -# fires these hooks; the canonical graphify-out/ belongs to the primary checkout, -# so rebuilding from a worktree is wasteful, writes a rogue delta-only graph the -# user never asked for, and races deploy/CI `git clean` against the detached -# rebuild ("failed to remove graphify-out/: Directory not empty") (#1809, #1806). -# A linked worktree has git-dir != git-common-dir. Both are resolved to absolute -# via `cd ... && pwd` before comparing: git's exported GIT_DIR / --git-dir can be -# absolute while --git-common-dir is the relative ".git", and a raw compare would -# false-positive on the PRIMARY checkout and wrongly skip it. -_WORKTREE_GUARD = """\ -_GFY_GITDIR=$(cd "$(git rev-parse --git-dir 2>/dev/null)" 2>/dev/null && pwd) -_GFY_COMMONDIR=$(cd "$(git rev-parse --git-common-dir 2>/dev/null)" 2>/dev/null && pwd) -if [ -n "$_GFY_COMMONDIR" ] && [ "$_GFY_GITDIR" != "$_GFY_COMMONDIR" ]; then - exit 0 -fi -""" - - -_HOOK_SCRIPT = """\ -# graphify-hook-start -# Auto-rebuilds the knowledge graph after each commit (code files only, no LLM needed). -# Installed by: graphify hook install -# Deterministic clustering: networkx louvain iterates string-keyed sets whose -# order is randomized per-process by PYTHONHASHSEED, so community assignments -# churn run-to-run. Pinning it makes graphify-out reproducible. -export PYTHONHASHSEED=0 - -# Git for Windows/MSYS hooks can inherit fragile pipe handles from GUI clients -# and agent shells. Keep hook-triggered rebuilds sequential by default there; -# explicit GRAPHIFY_MAX_WORKERS still wins for users who want parallelism. -if [ -n "${WINDIR:-}" ] || [ -n "${MSYSTEM:-}" ]; then - export GRAPHIFY_MAX_WORKERS="${GRAPHIFY_MAX_WORKERS:-1}" -fi - -# Skip during rebase/merge/cherry-pick to avoid blocking --continue with unstaged changes -# git exports GIT_DIR to hooks; the rev-parse fallback only runs when invoked by -# hand (each git exec costs 1s+ on AV-scanned Windows machines). -GIT_DIR=${GIT_DIR:-$(git rev-parse --git-dir 2>/dev/null)} -[ -d "$GIT_DIR/rebase-merge" ] && exit 0 -[ -d "$GIT_DIR/rebase-apply" ] && exit 0 -[ -f "$GIT_DIR/MERGE_HEAD" ] && exit 0 -[ -f "$GIT_DIR/CHERRY_PICK_HEAD" ] && exit 0 - -[ "${GRAPHIFY_SKIP_HOOK:-0}" = "1" ] && exit 0 - -""" + _WORKTREE_GUARD + """ -CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || git diff --name-only HEAD 2>/dev/null) -if [ -z "$CHANGED" ]; then - exit 0 -fi - -# Skip when only graphify-out/ artifacts changed (avoids rebuild loop when graph outputs are tracked in git) -_NON_GRAPH=$(echo "$CHANGED" | grep -v '^graphify-out/' || true) -if [ -z "$_NON_GRAPH" ]; then - exit 0 -fi - -""" + _PYTHON_DETECT + """ -export GRAPHIFY_CHANGED="$CHANGED" - -# Run the rebuild detached so git commit returns immediately. Full-repo rebuilds -# can take hours; blocking the post-commit hook stalls the shell. The Python -# launcher below detaches the child cross-platform, so it works on Git for -# Windows' shell too (which lacks the coreutils backgrounding tools) (#1161). -_GRAPHIFY_LOG="${HOME}/.cache/graphify-rebuild.log" -mkdir -p "$(dirname "$_GRAPHIFY_LOG")" -export GRAPHIFY_REBUILD_LOG="$_GRAPHIFY_LOG" -echo "[graphify hook] launching background rebuild (log: $_GRAPHIFY_LOG)" -""" + _detached_launch(_REBUILD_BODY_COMMIT) + """# graphify-hook-end -""" - - -_CHECKOUT_SCRIPT = """\ -# graphify-checkout-hook-start -# Auto-rebuilds the knowledge graph (code only) when switching branches. -# Installed by: graphify hook install - -# Deterministic clustering: networkx louvain iterates string-keyed sets whose -# order is randomized per-process by PYTHONHASHSEED, so community assignments -# churn run-to-run. Pinning it makes graphify-out reproducible. -export PYTHONHASHSEED=0 - -# Git for Windows/MSYS hooks can inherit fragile pipe handles from GUI clients -# and agent shells. Keep hook-triggered rebuilds sequential by default there; -# explicit GRAPHIFY_MAX_WORKERS still wins for users who want parallelism. -if [ -n "${WINDIR:-}" ] || [ -n "${MSYSTEM:-}" ]; then - export GRAPHIFY_MAX_WORKERS="${GRAPHIFY_MAX_WORKERS:-1}" -fi - -PREV_HEAD=$1 -NEW_HEAD=$2 -BRANCH_SWITCH=$3 - -# Only run on branch switches, not file checkouts -if [ "$BRANCH_SWITCH" != "1" ]; then - exit 0 -fi - -# Only run if graphify-out/ exists (graph has been built before) -if [ ! -d "graphify-out" ]; then - exit 0 -fi - -# Skip during rebase/merge/cherry-pick -# git exports GIT_DIR to hooks; the rev-parse fallback only runs when invoked by -# hand (each git exec costs 1s+ on AV-scanned Windows machines). -GIT_DIR=${GIT_DIR:-$(git rev-parse --git-dir 2>/dev/null)} -[ -d "$GIT_DIR/rebase-merge" ] && exit 0 -[ -d "$GIT_DIR/rebase-apply" ] && exit 0 -[ -f "$GIT_DIR/MERGE_HEAD" ] && exit 0 -[ -f "$GIT_DIR/CHERRY_PICK_HEAD" ] && exit 0 - -# Honor the same opt-out as post-commit: without this, GRAPHIFY_SKIP_HOOK=1 -# suppressed commit-triggered rebuilds but not branch-switch ones (#1809). -[ "${GRAPHIFY_SKIP_HOOK:-0}" = "1" ] && exit 0 - -""" + _WORKTREE_GUARD + _PYTHON_DETECT + """ -_GRAPHIFY_LOG="${HOME}/.cache/graphify-rebuild.log" -mkdir -p "$(dirname "$_GRAPHIFY_LOG")" -export GRAPHIFY_REBUILD_LOG="$_GRAPHIFY_LOG" -echo "[graphify] Branch switched - launching background rebuild (log: $_GRAPHIFY_LOG)" -""" + _detached_launch(_REBUILD_BODY_CHECKOUT) + """# graphify-checkout-hook-end -""" +_HOOK_MARKER = "# graphify-hook-v8" +_HOOK_END = "# /graphify-hook-v8" def _git_root(path: Path) -> Path | None: - """Walk up to find .git directory.""" - current = path.resolve() - for parent in [current, *current.parents]: - if (parent / ".git").exists(): - return parent - return None - - -_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]") - - -def _reject_windows_path(value: str, source: str) -> None: - """Raise if a hooks path looks like a Windows absolute path (#1385). - - On POSIX/WSL ``Path("C:\\Users\\...").is_absolute()`` is False, so an absolute - Windows hooks path gets joined under the repo root and mkdir'd as a literal - junk directory (backslashes and all), while install reports success and the - real ``.git/hooks`` gets nothing. Fail loudly instead so the user can fix it. - """ - if os.name == "nt": - return - if _WINDOWS_DRIVE_RE.match(value) or "\\" in value: - raise RuntimeError( - f"git hooks path from {source} looks like a Windows path: {value!r}. " - f"On WSL/POSIX this can't resolve to a real directory. Unset it with " - f"`git config --local --unset core.hooksPath`, or set a POSIX path." - ) + result = subprocess.run( + ["git", "-C", str(path), "rev-parse", "--show-toplevel"], + capture_output=True, text=True, check=False, + ) + value = result.stdout.strip() + return Path(value) if result.returncode == 0 and value else None def _hooks_dir(root: Path) -> Path: - """Return the git hooks directory, respecting core.hooksPath if set (e.g. Husky). - - Asks git itself via ``rev-parse --git-path hooks`` rather than parsing - ``.git/config`` with configparser: git legally allows duplicate keys and - sections (VS Code writes such configs), which a strict configparser rejects - with DuplicateOptionError/DuplicateSectionError, so every hook command - printed a spurious "could not read core.hooksPath" warning (#1907). git - resolves core.hooksPath, includeIf, and linked worktrees (where .git is a - file, not a directory) correctly in one place. Genuinely corrupt configs - are still surfaced: git itself fails on them, and its stderr is printed. - """ - # NOTE: do NOT pass --path-format=absolute — added in git 2.31; older git - # echoes it back as a literal argument, contaminating stdout and causing a - # phantom directory to be created (#907). git -C already returns an - # absolute path for worktree/external-gitdir cases, and a path relative to - # for normal repos — anchoring on root covers both. - import subprocess as _sp - try: - res = _sp.run( - ["git", "-C", str(root), "rev-parse", "--git-path", "hooks"], - capture_output=True, text=True, - ) - if res.returncode != 0: - # git failing here is a real signal (corrupt .git/config, tampering, - # permission flips by another tool). Surface git's own stderr rather - # than silently falling through to the default hooks directory. - err = (res.stderr or "").strip() - print( - f"[graphify hooks] git could not resolve the hooks path for " - f"{root}: {err or f'git exited with code {res.returncode}'}", - file=sys.stderr, + result = subprocess.run( + ["git", "-C", str(root), "config", "--get", "core.hooksPath"], + capture_output=True, text=True, check=False, + ) + configured = result.stdout.strip() + if configured: + path = Path(configured) + resolved = path if path.is_absolute() else root / path + return resolved.parent if resolved.name == "_" else resolved + common = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--git-common-dir"], + capture_output=True, text=True, check=True, + ).stdout.strip() + git_dir = Path(common) + if not git_dir.is_absolute(): + git_dir = root / git_dir + return git_dir / "hooks" + + +def _script(root: Path, checkout: bool) -> str: + python = sys.executable + if re.search(r"[^a-zA-Z0-9/_.@:\\-]", python): + python = "python3" + changed = "None" if checkout else "[Path(p) for p in subprocess.run(['git','diff-tree','--no-commit-id','--name-only','-r','HEAD'], capture_output=True, text=True).stdout.splitlines()]" + body = ( + "from pathlib import Path\n" + "import subprocess\n" + "from graphify.watch import _rebuild_code\n" + f"_rebuild_code(Path({str(root)!r}), changed_paths={changed})\n" + ) + return ( + "#!/bin/sh\n" + f"{_HOOK_MARKER}\n" + f"{shlex.quote(python)} -c {shlex.quote(body)}\n" + f"{_HOOK_END}\n" + ) + + +def _install_one(path: Path, content: str) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + existing = path.read_text(encoding="utf-8") + if _HOOK_MARKER in existing: + updated = re.sub( + rf"{re.escape(_HOOK_MARKER)}.*?{re.escape(_HOOK_END)}\n?", + content.split("\n", 1)[1], existing, flags=re.DOTALL, ) + path.write_text(updated, encoding="utf-8") else: - raw = res.stdout.strip() - # A valid hooks path can never contain newlines or NUL. Their presence - # means git echoed an unrecognised flag back (old git behaviour). - if raw and not any(c in raw for c in ("\n", "\r", "\x00")): - _reject_windows_path(raw, "git rev-parse --git-path hooks") - d = (root / raw).resolve() - d.mkdir(parents=True, exist_ok=True) - return d - except (OSError, FileNotFoundError): - pass - d = root / ".git" / "hooks" - d.mkdir(parents=True, exist_ok=True) - return d - - -def _install_hook(hooks_dir: Path, name: str, script: str, marker: str) -> str: - """Install a single git hook, appending if an existing hook is present.""" - hook_path = hooks_dir / name - if hook_path.exists(): - content = hook_path.read_text(encoding="utf-8") - if marker in content: - return f"already installed at {hook_path}" - hook_path.write_text(content.rstrip() + "\n\n" + script, encoding="utf-8", newline="\n") - return f"appended to existing {name} hook at {hook_path}" - hook_path.write_text("#!/bin/sh\n" + script, encoding="utf-8", newline="\n") - hook_path.chmod(0o755) - return f"installed at {hook_path}" - - -def _uninstall_hook(hooks_dir: Path, name: str, marker: str, marker_end: str) -> str: - """Remove graphify section from a git hook using start/end markers.""" - hook_path = hooks_dir / name - if not hook_path.exists(): - return f"no {name} hook found - nothing to remove." - content = hook_path.read_text(encoding="utf-8") - if marker not in content: - return f"graphify hook not found in {name} - nothing to remove." - new_content = re.sub( - rf"{re.escape(marker)}.*?{re.escape(marker_end)}\n?", - "", - content, - flags=re.DOTALL, - ).strip() - if not new_content or new_content in ("#!/bin/bash", "#!/bin/sh"): - hook_path.unlink() - return f"removed {name} hook at {hook_path}" - hook_path.write_text(new_content + "\n", encoding="utf-8", newline="\n") - return f"graphify removed from {name} at {hook_path} (other hook content preserved)" - - -def _pinned_python() -> str: - """Return sys.executable if its path is shell-safe, else an empty string. - - Applies the same allowlist used in _PYTHON_DETECT: rejects any character - that is not a valid plain filesystem path character, preventing $(...), - backtick, double-quote, semicolon, etc. from being injected into generated - shell scripts or the merge-driver command line. The allowlist includes ':' - and '\\' so Windows paths (C:\\...) are accepted. An empty return means - callers must fall back to the `graphify` launcher on PATH — safe degradation. - """ - if re.search(r"[^a-zA-Z0-9/_.@:\\-]", sys.executable): - return "" - return sys.executable - - -def _merge_attr_line() -> str: - """The .gitattributes line assigning the graphify merge driver to graph.json. - - The graph lives under the configured output directory (graphify.paths, - GRAPHIFY_OUT env override). gitattributes patterns are repo-relative, so an - absolute output-dir override cannot be expressed there — fall back to the - default name in that case. - """ - from graphify.paths import GRAPHIFY_OUT - out = GRAPHIFY_OUT - if not out or Path(out).is_absolute() or "\\" in out: - out = "graphify-out" - return f"{out.rstrip('/')}/graph.json merge=graphify" - - -def _has_merge_attr(content: str) -> bool: - """True if a (non-comment) `<...>graph.json ... merge=graphify` line exists.""" - for raw in content.splitlines(): - line = raw.strip() - if not line or line.startswith("#"): - continue - fields = line.split() - if fields and fields[0].endswith("graph.json") and "merge=graphify" in fields[1:]: - return True - return False - - -def _register_merge_driver(root: Path) -> str: - """Register the graph.json union merge driver in git config + .gitattributes (#1902). - - README and CHANGELOG 0.7.0 document `graphify merge-driver` as being set up - by `hook install`, but install never actually registered it. Writes go - through `git config` (never hand-edit .git/config — in a linked worktree the - effective config is not at root/.git/config). The interpreter is pinned the - same way the hook scripts pin it, so the driver works even when the graphify - launcher is not on PATH at merge time. - """ - import subprocess as _sp - pinned = _pinned_python() - if pinned: - driver = f"{pinned} -m graphify merge-driver %O %A %B" + path.write_text(existing.rstrip() + "\n" + content.split("\n", 1)[1], encoding="utf-8") else: - driver = "graphify merge-driver %O %A %B" - try: - for key, value in ( - ("merge.graphify.name", "graphify graph.json union merge"), - ("merge.graphify.driver", driver), - ): - _sp.run( - ["git", "-C", str(root), "config", key, value], - check=True, capture_output=True, text=True, - ) - except (OSError, _sp.CalledProcessError) as exc: - return f"not registered (git config failed: {exc})" - - line = _merge_attr_line() - attrs = root / ".gitattributes" - if attrs.exists(): - content = attrs.read_text(encoding="utf-8") - if _has_merge_attr(content): - return f"already registered ({line})" - # Never clobber other entries; preserve a trailing newline. - if content and not content.endswith("\n"): - content += "\n" - attrs.write_text(content + line + "\n", encoding="utf-8", newline="\n") - else: - attrs.write_text(line + "\n", encoding="utf-8", newline="\n") - return f"registered ({line})" - - -def _unregister_merge_driver(root: Path) -> str: - """Remove the merge-driver git config keys and the .gitattributes line.""" - import subprocess as _sp - for key in ("merge.graphify.name", "merge.graphify.driver"): - try: - # --unset exits nonzero if the key is absent; that is fine. - _sp.run( - ["git", "-C", str(root), "config", "--unset", key], - capture_output=True, text=True, - ) - except OSError: - pass - attrs = root / ".gitattributes" - if not attrs.exists(): - return "not registered - nothing to remove." - content = attrs.read_text(encoding="utf-8") - kept = [ - raw for raw in content.splitlines() - if not _has_merge_attr(raw) - ] - if kept == content.splitlines(): - return "gitattributes entry not found - nothing to remove." - if kept: - # Other entries survive; the file stays. - attrs.write_text("\n".join(kept) + "\n", encoding="utf-8", newline="\n") - return "removed from .gitattributes (other entries preserved)" - attrs.unlink() - return "removed (.gitattributes deleted - no other entries)" - - -def _merge_driver_status(root: Path) -> str: - """Report whether the merge driver is registered (config + gitattributes).""" - import subprocess as _sp - try: - res = _sp.run( - ["git", "-C", str(root), "config", "--get", "merge.graphify.driver"], - capture_output=True, text=True, - ) - cfg_ok = res.returncode == 0 and bool(res.stdout.strip()) - except OSError: - cfg_ok = False - attrs = root / ".gitattributes" - attr_ok = attrs.exists() and _has_merge_attr(attrs.read_text(encoding="utf-8")) - if cfg_ok and attr_ok: - return "registered" - if cfg_ok: - return "partially registered (git config set, .gitattributes line missing)" - if attr_ok: - return "partially registered (.gitattributes line set, git config missing)" - return "not registered" - - -def _user_hooks_dir(hooks_dir: Path) -> Path: - """Return the user-editable hooks directory. - - Husky 9 sets core.hooksPath to .husky/_ (wrapper scripts auto-generated by - Husky), while user-editable hooks live in the parent .husky/. Return the - parent when the resolved dir ends in '_' so install/status/uninstall target - the correct location (#987). - """ - if hooks_dir.name == "_": - return hooks_dir.parent - return hooks_dir + path.write_text(content, encoding="utf-8") + path.chmod(path.stat().st_mode | 0o111) + return f"installed at {path}" + + +def _uninstall_one(path: Path) -> str: + if not path.exists(): + return "not installed" + content = path.read_text(encoding="utf-8") + updated = re.sub( + rf"{re.escape(_HOOK_MARKER)}.*?{re.escape(_HOOK_END)}\n?", + "", content, flags=re.DOTALL, + ).strip() + if not updated or updated == "#!/bin/sh": + path.unlink() + return f"removed {path}" + path.write_text(updated + "\n", encoding="utf-8") + return f"removed Graphify section from {path}" def install(path: Path = Path(".")) -> str: - """Install graphify post-commit and post-checkout hooks in the nearest git repo.""" root = _git_root(path) if root is None: raise RuntimeError(f"No git repository found at or above {path.resolve()}") - - hooks_dir = _user_hooks_dir(_hooks_dir(root)) - - # Pin the current interpreter so the hook works even when the graphify - # launcher is not on PATH at git-trigger time (uv tool / pipx isolation). - # sys.executable is the Python running this very install command, so it is - # always the correct isolated-venv interpreter. The placeholder is replaced - # in both scripts before writing; the allowlist in _pinned_python() strips - # any characters unsafe in a shell path (empty result -> the pinned probe is - # skipped), and import-verification catches a stale pinned path so it safely - # falls through to the dynamic detection. - pinned = _pinned_python() - hook = _HOOK_SCRIPT.replace("__PINNED_PYTHON__", pinned) - checkout = _CHECKOUT_SCRIPT.replace("__PINNED_PYTHON__", pinned) - - commit_msg = _install_hook(hooks_dir, "post-commit", hook, _HOOK_MARKER) - checkout_msg = _install_hook(hooks_dir, "post-checkout", checkout, _CHECKOUT_MARKER) - merge_msg = _register_merge_driver(root) - - return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}\nmerge driver: {merge_msg}" + directory = _hooks_dir(root) + commit = _install_one(directory / "post-commit", _script(root, False)) + checkout = _install_one(directory / "post-checkout", _script(root, True)) + return f"post-commit: {commit}\npost-checkout: {checkout}" def uninstall(path: Path = Path(".")) -> str: - """Remove graphify post-commit and post-checkout hooks.""" root = _git_root(path) if root is None: raise RuntimeError(f"No git repository found at or above {path.resolve()}") - - hooks_dir = _user_hooks_dir(_hooks_dir(root)) - commit_msg = _uninstall_hook(hooks_dir, "post-commit", _HOOK_MARKER, _HOOK_MARKER_END) - checkout_msg = _uninstall_hook(hooks_dir, "post-checkout", _CHECKOUT_MARKER, _CHECKOUT_MARKER_END) - merge_msg = _unregister_merge_driver(root) - - return f"post-commit: {commit_msg}\npost-checkout: {checkout_msg}\nmerge driver: {merge_msg}" + directory = _hooks_dir(root) + return ( + f"post-commit: {_uninstall_one(directory / 'post-commit')}\n" + f"post-checkout: {_uninstall_one(directory / 'post-checkout')}" + ) def status(path: Path = Path(".")) -> str: - """Check if graphify hooks are installed.""" root = _git_root(path) if root is None: return "Not in a git repository." - hooks_dir = _user_hooks_dir(_hooks_dir(root)) + directory = _hooks_dir(root) + def state(name: str) -> str: + target = directory / name + return "installed" if target.exists() and _HOOK_MARKER in target.read_text(encoding="utf-8") else "not installed" + return f"post-commit: {state('post-commit')}\npost-checkout: {state('post-checkout')}" - def _check(name: str, marker: str) -> str: - p = hooks_dir / name - if not p.exists(): - return "not installed" - return "installed" if marker in p.read_text(encoding="utf-8") else "not installed (hook exists but graphify not found)" - commit = _check("post-commit", _HOOK_MARKER) - checkout = _check("post-checkout", _CHECKOUT_MARKER) - merge = _merge_driver_status(root) - return f"post-commit: {commit}\npost-checkout: {checkout}\nmerge driver: {merge}" +__all__ = ["install", "status", "uninstall"] diff --git a/graphify/impact.py b/graphify/impact.py new file mode 100644 index 000000000..86c36d040 --- /dev/null +++ b/graphify/impact.py @@ -0,0 +1,82 @@ +"""Pull-request/file impact analysis over an active Helix generation.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +from typing import Iterable + +from .helix.model import LoadedGraph, node_attributes + + +def changed_files(base: str, *, cwd: str | Path = ".") -> list[str]: + """Return files changed from ``base`` to HEAD without invoking a shell.""" + result = subprocess.run( + ["git", "diff", "--name-only", "--diff-filter=ACMRD", f"{base}...HEAD"], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + return [line for line in result.stdout.splitlines() if line] + + +def analyze_impact( + loaded: LoadedGraph, + files: Iterable[str | Path], + *, + depth: int = 2, +) -> dict: + """Return nodes, dependencies, and communities affected by source files.""" + graph = loaded.graph + requested = {Path(file).as_posix() for file in files} + requested_names = {Path(file).name for file in requested} + if loaded.query is None: + raise RuntimeError("impact analysis requires the native Helix query interface") + candidates = loaded.query.candidate_ids(sorted(requested | requested_names)) + seeds = { + node_id + for node_id in candidates + if (attrs := node_attributes(graph, node_id)) is not None + if (source := attrs.get("source_file")) + and ( + Path(str(source)).as_posix() in requested + or Path(str(source)).name in requested_names + or any(Path(str(source)).as_posix().endswith(f"/{item}") for item in requested) + ) + } + impacted = set(seeds) + traversed: list[tuple] = [] + if seeds: + from helixdb.graph import TraversalOptions + + result = graph.traverse(TraversalOptions( + seeds=tuple(sorted(seeds, key=repr)), + max_depth=max(0, depth), + direction="both", + )) + impacted = {visit.node_id for visit in result.visits} + traversed = [(edge.source, edge.target) for edge in result.discovery_edges] + + membership = { + member: record.get("id") + for record in loaded.state.get("communities", []) + if isinstance(record, dict) + for member in record.get("members", []) + } + communities = sorted({ + community_id + for node in impacted + if isinstance((community_id := membership.get(node)), int) + }) + return { + "changed_files": sorted(requested), + "seed_nodes": sorted(seeds, key=str), + "impacted_nodes": sorted(impacted, key=str), + "impacted_communities": communities, + "traversed_edges": [list(edge) for edge in traversed], + "depth": depth, + } + + +__all__ = ["analyze_impact", "changed_files"] diff --git a/graphify/install.py b/graphify/install.py index e2b3a5cdb..c3e3f0fbe 100644 --- a/graphify/install.py +++ b/graphify/install.py @@ -521,11 +521,6 @@ def _print_banner() -> None: if not sys.stdout.isatty(): return try: - if sys.platform == "win32": - import ctypes - ctypes.windll.kernel32.SetConsoleMode( - ctypes.windll.kernel32.GetStdHandle(-11), 7 - ) A = "\033[38;5;214m" D = "\033[38;5;130m" R = "\033[0m" @@ -938,7 +933,7 @@ def _antigravity_install(project_dir: Path) -> None: print(' "graphify": {') print(' "command": "uv",') print( - ' "args": ["run", "--with", "graphifyy", "--with", "mcp", "-m", "graphify.serve", "${workspace.path}/graphify-out/graph.json"]' + ' "args": ["run", "--with", "graphifyy", "--with", "mcp", "-m", "graphify.serve", "${workspace.path}/graphify-out/graph.helix"]' ) print(" }") def _antigravity_uninstall(project_dir: Path, *, project: bool = False) -> None: @@ -995,7 +990,7 @@ def _antigravity_uninstall(project_dir: Path, *, project: bool = False) -> None: Only use Read/Grep/Glob directly when: 1. graphify has already oriented you and you need to modify or debug specific lines -2. `graphify-out/graph.json` does not exist yet +2. `graphify-out/graph.helix` does not exist yet - If `graphify-out/wiki/index.md` exists, navigate it instead of reading raw files - Read `graphify-out/GRAPH_REPORT.md` only for broad architecture review when query/path/explain do not surface enough context @@ -1033,7 +1028,7 @@ def _cursor_uninstall(project_dir: Path) -> None: This project has a graphify knowledge graph at graphify-out/. Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- For codebase or architecture questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. - If graphify-out/wiki/index.md exists, navigate it instead of reading raw files - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context - After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) @@ -1067,7 +1062,7 @@ def _devin_rules_uninstall(project_dir: Path) -> None: return { "tool.execute.before": async (input, output) => { if (reminded) return; - if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; + if (!existsSync(join(directory, "graphify-out", "graph.helix"))) return; if (input.tool === "bash") { // Separate with ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a @@ -1214,7 +1209,7 @@ def _uninstall_kilo_plugin(project_dir: Path) -> None: f" {write_config_file.relative_to(project_dir)} -> plugin deregistered" ) # OpenCode tool.execute.before plugin — fires before every tool call. -# Injects a graph reminder into bash command output when graph.json exists. +# Injects a graph reminder into bash command output when graph.helix exists. _OPENCODE_PLUGIN_JS = """\ // graphify OpenCode plugin // Injects a knowledge graph reminder before bash tool calls when the graph exists. @@ -1233,7 +1228,7 @@ def _uninstall_kilo_plugin(project_dir: Path) -> None: return { "tool.execute.before": async (input, output) => { if (reminded) return; - if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; + if (!existsSync(join(directory, "graphify-out", "graph.helix"))) return; if (input.tool === "bash") { // ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a statement diff --git a/graphify/llm.py b/graphify/llm.py index 48546b865..abad8accc 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -567,7 +567,7 @@ def _read_files(units: "list[Path | FileSlice]", root: Path) -> str: # slips through it. This closes that intra-file gap with a lenient substring # check and FLAGS (never drops) an unverifiable node with ``verification = # "unverified"``, surfaced by the caller (stderr), reported by the diagnostics, -# and left on the node in graph.json. +# and left on the node in native graph. # Short tokens (len < 3) are ignored: they match too readily to be evidence and # their absence is not a reliable fabrication signal, so skipping them avoids # false positives. @@ -1342,7 +1342,7 @@ def _call_claude_cli(user_message: str, max_tokens: int = 8192, *, deep_mode: bo # attached — what would you like me to do with it?"). That prose parses to # zero nodes/edges, so _response_is_hollow flags it as truncation and the # adaptive-retry path bisects the chunk indefinitely, never converging and - # never writing graph.json (verified against Claude Code 2.1.197). + # never writing native graph (verified against Claude Code 2.1.197). # # Putting the full extraction schema plus an explicit imperative in the # user turn — and dropping --system-prompt — makes the CLI emit the JSON @@ -1831,7 +1831,7 @@ def _strip_partial_markers(result: dict) -> None: Call this only AFTER the semantic cache has been saved (the save consumes the marker to stamp affected entries ``partial: True``). Stripping it keeps the - internal flag out of the graph.json nodes/edges the corpus result feeds into. + internal flag out of the native graph nodes/edges the corpus result feeds into. """ for bucket in ("nodes", "edges", "hyperedges"): for item in result.get(bucket, []): @@ -2044,7 +2044,8 @@ def extract_corpus_parallel( max_concurrency: int = 4, max_retry_depth: int = 3, deep_mode: bool = False, - cache_root: "Path | None" = None, + cache: dict | None = None, + cache_root: Path | None = None, ) -> dict: """Extract a corpus in chunks, merging results. @@ -2080,14 +2081,6 @@ def extract_corpus_parallel( output_tokens. Failed chunks are logged to stderr and skipped — one bad chunk does not abort the run. - ``cache_root`` (when given) is where per-chunk checkpoint cache entries are - written, decoupled from ``root`` which anchors content-hash keys and - ``source_file`` resolution — the same split the AST cache uses (#1774). - With ``--out``, cli.py passes the corpus as ``root`` and the output - directory as ``cache_root`` so checkpoints land where the recovery read - looks, instead of creating an unwanted ``graphify-out/`` inside the - analyzed source tree (#1990). - Accepts ``str`` paths as well as ``Path``; string entries are coerced up front so packing/slicing helpers can rely on ``Path`` semantics (#1386). """ @@ -2175,6 +2168,7 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # authoritative: pass the partial file set so its entry is # stamped ``partial: True`` and re-dispatched next run. partial_source_files=_partial_source_files(result) or None, + cache=cache, ) except Exception as _exc: # noqa: BLE001 — checkpoint is best-effort print(f"[graphify] incremental cache checkpoint failed: {_exc}", file=sys.stderr) @@ -2197,7 +2191,7 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: else: # Merge in deterministic submission order, NOT completion order. Merging # as chunks finish makes the node/edge ordering in the returned corpus - # (and therefore graph.json) depend on which network call happened to + # (and therefore native graph) depend on which network call happened to # return first — so identical input churned run-to-run (#1632). Collect # results keyed by chunk index and merge in sorted order after the pool # drains; this matches the serial path's order. The progress callback @@ -2242,7 +2236,7 @@ def _checkpoint_chunk(result: dict, chunk: "list[Path | FileSlice]") -> None: # Out-of-scope node filter (#1895). The #1757 cache guard already refuses # to WRITE a cache entry for a node whose source_file is a real file that # was not dispatched, but the node itself still flowed into the merged - # result and landed in graph.json. Mirror the #1757 condition here: resolve + # result and landed in native graph. Mirror the #1757 condition here: resolve # each source_file against root and drop the node only when it resolves to # an existing file (.is_file()) outside the dispatched set — non-file # source_files (concepts, model-invented anchors) pass through untouched. @@ -2661,6 +2655,8 @@ def _community_label_lines(G, communities, gods, max_communities, top_k): representative node labels (god nodes first). Returns (lines, labeled_cids); skips communities with no resolvable nodes.""" # gods may be node-id strings or god_nodes() dicts ({"id": ..., "label": ...}). + from graphify.helix.model import node_attributes + god_set = {g["id"] if isinstance(g, dict) else g for g in (gods or [])} ordered = sorted(communities.items(), key=lambda kv: -len(kv[1])) lines: list[str] = [] @@ -2670,7 +2666,11 @@ def _community_label_lines(G, communities, gods, max_communities, top_k): names: list[str] = [] seen: set[str] = set() for nid in ranked: - label = str(G.nodes[nid].get("label", nid)) if nid in G.nodes else str(nid) + label = ( + str(node_attributes(G, nid).get("label", nid)) + if G.contains_node(nid) + else str(nid) + ) label = label.strip().strip("()")[:_LABEL_MAXLEN] if label and label.lower() not in seen: seen.add(label.lower()) diff --git a/graphify/manifest_ingest.py b/graphify/manifest_ingest.py index ae3aa61fc..55ee3f1a4 100644 --- a/graphify/manifest_ingest.py +++ b/graphify/manifest_ingest.py @@ -16,7 +16,7 @@ from __future__ import annotations import re -import xml.etree.ElementTree as ET +from defusedxml import ElementTree as ET from pathlib import Path from typing import Any @@ -92,7 +92,7 @@ def extract_package_manifest(path: Path) -> dict[str, Any]: seen.add(dep_nid) # The edge targets the dependency's canonical package id. If that package's # own manifest is in the corpus, the edge resolves to its (single) node; if - # the dependency is external, build_from_json prunes the dangling edge. We + # the dependency is external, build_from_extraction prunes the dangling edge. We # deliberately do NOT emit a stub node — a stub with an empty source_file # would risk clobbering the real node's source_file under id-dedup. edges.append({ diff --git a/graphify/multigraph_compat.py b/graphify/multigraph_compat.py deleted file mode 100644 index 7ac62e275..000000000 --- a/graphify/multigraph_compat.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Runtime compatibility probe for Graphify MultiDiGraph mode. - -Verifies that the current NetworkX runtime supports the behaviors a future -opt-in --multigraph build will rely on. The probe is BEHAVIOR-based, not -version-based — both NX 3.4.2 (Py 3.10 lane) and NX 3.6.1+ (Py 3.11+ lane) -pass. The probe result is cached for the process lifetime via lru_cache. - -No call sites added yet; downstream multigraph PRs will gate on -require_multigraph_capabilities() before enabling MDG mode. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from functools import lru_cache -import sys -from typing import Any - -import networkx as nx -from networkx.readwrite import json_graph - - -@dataclass(frozen=True) -class CapabilityCheck: - name: str - ok: bool - detail: str - - -@dataclass(frozen=True) -class MultigraphCapabilityResult: - python_version: str - networkx_version: str - checks: tuple[CapabilityCheck, ...] - - @property - def ok(self) -> bool: - return all(check.ok for check in self.checks) - - @property - def failed(self) -> tuple[CapabilityCheck, ...]: - return tuple(check for check in self.checks if not check.ok) - - def error_message(self) -> str: - if self.ok: - return ( - "Graphify MultiDiGraph capability probe passed " - f"(Python {self.python_version}, NetworkX {self.networkx_version})." - ) - failed = "; ".join(f"{check.name}: {check.detail}" for check in self.failed) - return ( - "error: --multigraph requires NetworkX keyed MultiDiGraph node-link " - "round-trip support. " - f"Detected Python {self.python_version}, NetworkX {self.networkx_version}. " - f"Failed capability check(s): {failed}. " - "Default simple graph mode remains available." - ) - - -def _check(name: str, func: Callable[[], bool | str]) -> CapabilityCheck: - try: - detail = func() - except Exception as exc: - return CapabilityCheck(name, False, f"{type(exc).__name__}: {exc}") - if detail is True: - return CapabilityCheck(name, True, "ok") - if isinstance(detail, str): - return CapabilityCheck(name, False, detail) - return CapabilityCheck(name, False, f"unexpected result {detail!r}") - - -def _build_probe_graph() -> nx.MultiDiGraph: - graph = nx.MultiDiGraph() - graph.add_node("a", label="A") - graph.add_node("b", label="B") - graph.add_edge("a", "b", key="calls:a.py:L1", relation="calls", source_file="a.py") - graph.add_edge("a", "b", key="imports:a.py:L2", relation="imports", source_file="a.py") - return graph - - -def _probe_keyed_parallel_edges() -> bool | str: - graph = _build_probe_graph() - if not graph.is_multigraph() or not graph.is_directed(): - return f"probe graph type was {type(graph).__name__}" - if graph.number_of_edges("a", "b") != 2: - return f"expected 2 keyed parallel edges, got {graph.number_of_edges('a', 'b')}" - keys = set(graph["a"]["b"].keys()) - expected = {"calls:a.py:L1", "imports:a.py:L2"} - if keys != expected: - return f"expected keys {sorted(expected)}, got {sorted(keys)}" - return True - - -def _probe_node_link_round_trip() -> bool | str: - graph = _build_probe_graph() - data = json_graph.node_link_data(graph, edges="links") - if data.get("multigraph") is not True: - return f"serialized multigraph flag was {data.get('multigraph')!r}" - if data.get("directed") is not True: - return f"serialized directed flag was {data.get('directed')!r}" - links = data.get("links") - if not isinstance(links, list) or len(links) != 2: - length = 0 if not isinstance(links, list) else len(links) - return f"serialized links length was {length}" - serialized_keys: set[str] = set() - for edge in links: - if isinstance(edge, dict): - edge_key = edge.get("key") - if isinstance(edge_key, str): - serialized_keys.add(edge_key) - expected = {"calls:a.py:L1", "imports:a.py:L2"} - if serialized_keys != expected: - return f"serialized keys {sorted(serialized_keys)} did not match {sorted(expected)}" - loaded = json_graph.node_link_graph(data, edges="links") - if not isinstance(loaded, nx.MultiDiGraph): - return f"round-trip graph type was {type(loaded).__name__}" - if loaded.number_of_edges("a", "b") != 2: - return f"round-trip edge count was {loaded.number_of_edges('a', 'b')}" - loaded_keys = set(loaded["a"]["b"].keys()) - if loaded_keys != expected: - return f"round-trip keys {sorted(loaded_keys)} did not match {sorted(expected)}" - return True - - -def _probe_duplicate_key_overwrite_semantics() -> bool | str: - graph = nx.MultiDiGraph() - graph.add_edge("x", "y", key="same", marker="first") - graph.add_edge("x", "y", key="same", marker="second") - edges = list(graph.edges(keys=True, data=True)) - if len(edges) != 1: - return f"expected one edge after duplicate-key add, got {len(edges)}" - if edges[0][3].get("marker") != "second": - return f"expected second attr overwrite, got {edges[0][3].get('marker')!r}" - return True - - -def _probe_reserved_key_attr_rejected() -> bool | str: - """Verify the Python language guarantee that NetworkX add_edge inherits. - - Python forbids passing the same keyword argument twice — once explicitly - and once via **kwargs. This probe confirms that protection still applies - to nx.MultiDiGraph.add_edge: a future loader that builds attrs from JSON - will be reliably protected from accidentally setting `key` via attrs while - also passing `key=` explicitly. - - The probe always passes on any Python 3.x version. Its purpose is to - document the invariant explicitly in the probe suite so that if a future - Python version relaxes this rule (extremely unlikely), the probe surfaces - the regression. - """ - graph = nx.MultiDiGraph() - attrs: dict[str, Any] = {"key": "attr-key", "relation": "calls"} - try: - graph.add_edge("a", "b", key="schema-key", **attrs) - except TypeError: - return True - return "add_edge accepted duplicate key keyword and attr; loader must not rely on this" - - -def _probe_remove_edges_from_two_tuple_semantics() -> bool | str: - graph = nx.MultiDiGraph() - graph.add_edge("a", "b", key="one") - graph.add_edge("a", "b", key="two") - graph.remove_edges_from([("a", "b")]) - remaining = graph.number_of_edges("a", "b") - if remaining != 1: - return f"expected one remaining edge after two-tuple removal, got {remaining}" - return True - - -def _probe_to_undirected_preserves_multigraph_type() -> bool | str: - graph = _build_probe_graph() - undirected = graph.to_undirected() - undirected_view = graph.to_undirected(as_view=True) - if not isinstance(undirected, nx.MultiGraph): - return f"to_undirected() returned {type(undirected).__name__}" - if not isinstance(undirected_view, nx.MultiGraph): - return f"to_undirected(as_view=True) returned {type(undirected_view).__name__}" - return True - - -@lru_cache(maxsize=1) -def probe_multigraph_capabilities() -> MultigraphCapabilityResult: - checks = ( - _check("keyed_parallel_edges", _probe_keyed_parallel_edges), - _check("node_link_edges_links_round_trip", _probe_node_link_round_trip), - _check("duplicate_key_overwrite_semantics", _probe_duplicate_key_overwrite_semantics), - _check("reserved_key_attr_rejected", _probe_reserved_key_attr_rejected), - _check( - "remove_edges_from_two_tuple_semantics", - _probe_remove_edges_from_two_tuple_semantics, - ), - _check( - "to_undirected_preserves_multigraph_type", - _probe_to_undirected_preserves_multigraph_type, - ), - ) - return MultigraphCapabilityResult( - python_version=( - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ), - networkx_version=nx.__version__, - checks=checks, - ) - - -def require_multigraph_capabilities() -> MultigraphCapabilityResult: - result = probe_multigraph_capabilities() - if not result.ok: - raise RuntimeError(result.error_message()) - return result diff --git a/graphify/operations.py b/graphify/operations.py new file mode 100644 index 000000000..4956a8128 --- /dev/null +++ b/graphify/operations.py @@ -0,0 +1,145 @@ +"""Atomic operations over an existing active Helix generation.""" + +from __future__ import annotations + +import copy +from pathlib import Path + +from .analyze import god_nodes, surprising_connections, suggest_questions +from .cluster import cluster, score_all +from .helix.persistence import DEFAULT_PROJECT_STORE, HelixEmbeddedStore +from .helix.state import ( + communities_from_state, + community_records, + community_summaries, + labels_from_state, +) + + +def recluster(store_path: str | Path = DEFAULT_PROJECT_STORE) -> dict[int, list]: + """Replace community records and node membership in one generation.""" + with HelixEmbeddedStore(store_path) as store: + loaded = store.load() + graph = loaded.graph + previous_state = dict(loaded.state) + state = copy.deepcopy(previous_state) + communities = cluster(graph) + cohesion = score_all(graph, communities) + previous_labels = labels_from_state(state) + labels = { + community_id: previous_labels.get( + community_id, f"Community {community_id}" + ) + for community_id in communities + } + state["communities"] = community_records( + communities, + labels=labels, + cohesion=cohesion, + naming_source="preserved" if previous_labels else "generated", + ) + store.replace_state( + state, previous_state=previous_state, snapshot=loaded + ) + return communities + + +def reanalyze(store_path: str | Path = DEFAULT_PROJECT_STORE) -> dict: + """Replace analysis records while preserving topology and other state.""" + with HelixEmbeddedStore(store_path) as store: + loaded = store.load() + graph = loaded.graph + previous_state = dict(loaded.state) + state = copy.deepcopy(previous_state) + communities = communities_from_state(state) + labels = labels_from_state(state) + analysis = state.get("analysis", {}) + report_inputs = analysis.get("report_inputs", {}) if isinstance(analysis, dict) else {} + refreshed = { + "god_nodes": god_nodes(graph), + "surprises": surprising_connections(graph, communities), + "suggested_questions": suggest_questions(graph, communities, labels), + "community_summaries": community_summaries(graph, communities, labels), + "report_inputs": report_inputs, + } + state["analysis"] = refreshed + store.replace_state( + state, previous_state=previous_state, snapshot=loaded + ) + return refreshed + + +def relabel( + store_path: str | Path = DEFAULT_PROJECT_STORE, + *, + backend: str | None = None, + model: str | None = None, + missing_only: bool = False, + max_concurrency: int = 4, + batch_size: int = 100, +) -> dict[int, str]: + """Name native communities and activate the labels with the same topology.""" + from .cluster import label_communities_by_hub + from .llm import detect_backend, label_communities + + with HelixEmbeddedStore(store_path) as store: + loaded = store.load() + graph = loaded.graph + previous_state = dict(loaded.state) + state = copy.deepcopy(previous_state) + communities = communities_from_state(state) + previous = labels_from_state(state) + selected = backend or detect_backend() + to_label = { + cid: members + for cid, members in communities.items() + if not ( + missing_only + and cid in previous + and not previous[cid].startswith("Community ") + ) + } + if selected and to_label: + generated = label_communities( + graph, + to_label, + backend=selected, + model=model, + gods=state.get("analysis", {}).get("god_nodes", []), + max_concurrency=max_concurrency, + batch_size=batch_size, + ) + elif to_label: + generated = label_communities_by_hub(graph, to_label) + else: + generated = {} + labels = { + cid: ( + previous[cid] + if missing_only + and cid in previous + and not previous[cid].startswith("Community ") + else generated.get(cid, f"Community {cid}") + ) + for cid in communities + } + cohesion = { + record["id"]: float(record["cohesion"]) + for record in state.get("communities", []) + if isinstance(record, dict) + and isinstance(record.get("id"), int) + and isinstance(record.get("cohesion"), (int, float)) + } + state["communities"] = community_records( + communities, + labels=labels, + cohesion=cohesion, + naming_source=selected or "native-hub", + ) + store.replace_state( + state, previous_state=previous_state, snapshot=loaded + ) + return labels + + +__all__ = ["reanalyze", "recluster", "relabel"] diff --git a/graphify/paths.py b/graphify/paths.py index d43bc7766..05317f67d 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -294,11 +294,6 @@ def out_path(*parts: str) -> Path: return Path(GRAPHIFY_OUT, *parts) -def default_graph_json() -> str: - """Default ``graph.json`` path under the configured output dir. - - The package-wide fallback used by serve/build/benchmark/prs and the CLI read - commands so a ``GRAPHIFY_OUT`` override is honoured everywhere, not just where - the path is passed explicitly (#1423). - """ - return str(out_path("graph.json")) +def default_graph_store() -> str: + """Default embedded Helix store under the configured output directory.""" + return str(out_path("graph.helix")) diff --git a/graphify/prs.py b/graphify/prs.py index 9534e6c00..5adbdf843 100644 --- a/graphify/prs.py +++ b/graphify/prs.py @@ -25,12 +25,10 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from typing import TYPE_CHECKING +from typing import Any -if TYPE_CHECKING: - import networkx as nx - -from graphify.paths import default_graph_json as _default_graph_json +from graphify.helix.model import LoadedGraph, graphify_attributes +from graphify.helix.persistence import DEFAULT_PROJECT_STORE, load_graph # ── ANSI colours ───────────────────────────────────────────────────────────── @@ -73,7 +71,7 @@ class PRInfo: updated_at: datetime expected_base: str = "main" # set by fetch_prs via _detect_default_branch worktree_path: str | None = None - # Graph impact — populated when graph.json exists + # Graph impact — populated when a Helix store exists communities_touched: list[int] = field(default_factory=list) nodes_affected: int = 0 files_changed: list[str] = field(default_factory=list) @@ -161,8 +159,10 @@ def _detect_default_branch(repo: str | None = None) -> str: if repo: args += ["--repo", repo] data = _gh(*args) - if data and data.get("defaultBranchRef", {}).get("name"): - return data["defaultBranchRef"]["name"] + if isinstance(data, dict): + branch = data.get("defaultBranchRef") + if isinstance(branch, dict) and isinstance(branch.get("name"), str): + return branch["name"] # Fall back to git symbolic-ref for the current repo try: result = subprocess.run( @@ -240,7 +240,7 @@ def fetch_pr_files(number: int, repo: str | None = None) -> list[str]: return [] -# ── Graph-native impact (used by MCP tools — works on nx.Graph directly) ───── +# ── Graph-native impact (used by MCP tools) ────────────────────────────────── def _path_match(graph_src: str, pr_file: str) -> bool: """True if graph_src and pr_file refer to the same file (path-boundary safe).""" @@ -249,7 +249,13 @@ def _path_match(graph_src: str, pr_file: str) -> bool: return graph_src.endswith("/" + pr_file) or pr_file.endswith("/" + graph_src) -def compute_pr_impact(files: list[str], G: "nx.Graph") -> tuple[list[int], int]: +def compute_pr_impact( + files: list[str], + G: Any, + communities: dict[int, list[Any]] | None = None, + *, + native_query: Any, +) -> tuple[list[int], int]: """Return (communities_touched, nodes_affected) for a set of changed files. Builds a file→(communities, count) index first so lookup is O(nodes + files) @@ -258,14 +264,23 @@ def compute_pr_impact(files: list[str], G: "nx.Graph") -> tuple[list[int], int]: # Build index once file_comms: dict[str, set[int]] = {} file_count: dict[str, int] = {} - for _, data in G.nodes(data=True): + membership = { + node_id: cid + for cid, members in (communities or {}).items() + for node_id in members + } + for node_id in native_query.candidate_ids(files): + node = G.node(node_id) + if node is None: + continue + data = graphify_attributes(node.attributes) src = data.get("source_file") or "" if not src: continue if src not in file_comms: file_comms[src] = set() file_count[src] = 0 - c = data.get("community") + c = membership.get(node_id, data.get("community")) if c is not None: file_comms[src].add(int(c)) file_count[src] += 1 @@ -324,27 +339,38 @@ def fetch_worktrees() -> dict[str, str]: # ── Graph impact analysis ───────────────────────────────────────────────────── -def _load_graph_json(graph_path: Path) -> dict | None: +def _load_graph_store(graph_path: Path) -> LoadedGraph | None: if not graph_path.exists(): return None - from graphify.security import check_graph_file_size_cap try: - check_graph_file_size_cap(graph_path) - return json.loads(graph_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError, ValueError): + return load_graph(graph_path) + except (OSError, RuntimeError, ValueError): return None -def build_community_labels(data: dict, top_n: int = 4) -> dict[int, list[str]]: - """Return {community_id: [top_labels]} extracted from graph node data.""" +def build_community_labels( + loaded: LoadedGraph | dict[str, Any], top_n: int = 4 +) -> dict[int, list[str]]: + """Return community IDs mapped to representative native node labels.""" comm_labels: dict[int, list[str]] = defaultdict(list) - for node in data.get("nodes", []): - c = node.get("community") - if c is None: + if isinstance(loaded, dict): + for node in loaded.get("nodes", []): + if not isinstance(node, dict) or node.get("community") is None: + continue + label = node.get("label") or node.get("id") + if label: + comm_labels[int(node["community"])].append(str(label)) + return {cid: labels[:top_n] for cid, labels in comm_labels.items()} + for record in loaded.state.get("communities", []): + if not isinstance(record, dict) or not isinstance(record.get("id"), int): continue - label = node.get("label") or node.get("id") or "" - if label: - comm_labels[int(c)].append(label) + cid = record["id"] + for node_id in record.get("members", []): + node = loaded.graph.node(node_id) + if node is None: + continue + attrs = graphify_attributes(node.attributes) + comm_labels[cid].append(str(attrs.get("label") or node_id)) return {c: labels[:top_n] for c, labels in comm_labels.items()} @@ -352,25 +378,10 @@ def attach_graph_impact( prs: list[PRInfo], graph_path: Path, repo: str | None = None ) -> dict[int, list[str]]: """Fetch PR file lists concurrently, compute graph impact, return community labels.""" - data = _load_graph_json(graph_path) - if not data: + loaded = _load_graph_store(graph_path) + if loaded is None: return {} - # Build file → {community, node_count} index - file_to_communities: dict[str, set[int]] = {} - file_to_nodes: dict[str, int] = {} - for node in data.get("nodes", []): - src = node.get("source_file") or "" - if not src: - continue - comm = node.get("community") - if src not in file_to_communities: - file_to_communities[src] = set() - file_to_nodes[src] = 0 - if comm is not None: - file_to_communities[src].add(int(comm)) - file_to_nodes[src] += 1 - # Fetch diffs concurrently — gh pr diff is the bottleneck (network I/O) actionable = [pr for pr in prs if pr.status != "WRONG-BASE"] workers = min(8, len(actionable)) if actionable else 1 @@ -387,19 +398,62 @@ def attach_graph_impact( files = [] pr.files_changed = files - comms: set[int] = set() - nodes = 0 - matched: set[str] = set() - for f in files: - for gf, gcomms in file_to_communities.items(): - if gf not in matched and _path_match(gf, f): - comms |= gcomms - nodes += file_to_nodes.get(gf, 0) - matched.add(gf) - pr.communities_touched = sorted(comms) - pr.nodes_affected = nodes + # Project only nodes matching changed paths through public Helix predicates. + # Chunk the OR terms so a PR touching many files cannot create an unbounded + # query plan, then use native point reads for exact path matching. + if loaded.query is None: + raise RuntimeError("PR impact requires the native Helix query interface") + changed_files = sorted({path for pr in actionable for path in pr.files_changed}) + terms = list(dict.fromkeys( + value + for path in changed_files + for value in (path, Path(path).name) + if value + )) + candidate_ids: list[Any] = [] + seen_candidates: set[Any] = set() + for offset in range(0, len(terms), 64): + for node_id in loaded.query.candidate_ids(terms[offset : offset + 64]): + if node_id not in seen_candidates: + seen_candidates.add(node_id) + candidate_ids.append(node_id) + + file_to_communities: dict[str, set[int]] = {} + file_to_nodes: dict[str, int] = {} + communities = { + record["id"]: list(record.get("members", [])) + for record in loaded.state.get("communities", []) + if isinstance(record, dict) and isinstance(record.get("id"), int) + } + membership = {node: cid for cid, members in communities.items() for node in members} + for node_id in candidate_ids: + node = loaded.graph.node(node_id) + if node is None: + continue + attrs = graphify_attributes(node.attributes) + src = str(attrs.get("source_file") or "") + if not src or not any(_path_match(src, path) for path in changed_files): + continue + comm = membership.get(node_id) + file_to_communities.setdefault(src, set()) + file_to_nodes[src] = file_to_nodes.get(src, 0) + 1 + if comm is not None: + file_to_communities[src].add(int(comm)) + + for pr in actionable: + comms: set[int] = set() + nodes = 0 + matched: set[str] = set() + for path in pr.files_changed: + for graph_file, graph_communities in file_to_communities.items(): + if graph_file not in matched and _path_match(graph_file, path): + comms |= graph_communities + nodes += file_to_nodes.get(graph_file, 0) + matched.add(graph_file) + pr.communities_touched = sorted(comms) + pr.nodes_affected = nodes - return build_community_labels(data) + return build_community_labels(loaded) # ── Dashboard rendering ─────────────────────────────────────────────────────── @@ -494,7 +548,7 @@ def render_conflicts( ) -> None: actionable = [p for p in prs if p.base_branch == base and p.communities_touched] if not actionable: - print(dim("\n No graph impact data - run with a valid graph.json to detect conflicts.\n")) + print(dim("\n No graph impact data - build a valid graph.helix store to detect conflicts.\n")) return # Build community → [PRs] map @@ -686,7 +740,7 @@ def cmd_prs(argv: list[str]) -> None: do_conflicts = False show_wrong_base = False pr_number: int | None = None - graph_path = Path(_default_graph_json()) + graph_path = DEFAULT_PROJECT_STORE i = 0 while i < len(argv): diff --git a/graphify/reflect.py b/graphify/reflect.py index 4e04e0589..ec0a96fe4 100644 --- a/graphify/reflect.py +++ b/graphify/reflect.py @@ -17,7 +17,7 @@ trusted lesson. When a graph is in hand, source nodes that no longer exist are dropped. It is deterministic: no LLM, stable sort orders, byte-stable output for a given input -and a given ``now``. When a graph (`graph.json` + `.graphify_analysis.json`) is available +and a given ``now``. When a Helix generation is available the lessons are also grouped by community label; without it they degrade to a single flat section. @@ -39,11 +39,6 @@ _UNCATEGORIZED = "Uncategorized" -# Derived experiential layer written alongside graph.json (a SIDECAR, kept -# separate from the durable structural truth in graph.json — no learning_* -# fields are ever stamped into the graph itself). Read-surface annotations are -# merged in at display time from this file. -LEARNING_SIDECAR_NAME = ".graphify_learning.json" _LEARNING_SCHEMA_VERSION = 1 _PROVENANCE_CAP = 5 # most-recent (question, date, outcome) entries per node @@ -159,52 +154,30 @@ def load_memory_docs(memory_dir: Path) -> list[dict[str, Any]]: # --- graph / community lookup (optional) --------------------------------------- -def _load_node_community(graph_path: Path, analysis_path: Path, - labels_path: Path) -> dict[str, str] | None: +def _load_node_community(graph_path: Path) -> dict[str, str] | None: """Build a lookup from node id AND node label -> community label, or None if the graph isn't available. - Mirrors how `graphify export wiki` reads graph.json + .graphify_analysis.json + - .graphify_labels.json. Community membership in the analysis sidecar is keyed by - node id, but `save-result` cites nodes by label, so both are mapped — otherwise a - cited ``build_from_json()`` never finds its community and every lesson collapses - into Uncategorized. Best-effort: any missing/unparseable artifact disables grouping. + Community state and topology are read from the same active generation. """ - if not graph_path.exists() or not analysis_path.exists(): - return None try: - analysis = json.loads(analysis_path.read_text(encoding="utf-8")) - except (OSError, ValueError): - return None - communities = analysis.get("communities", {}) - if not communities: + from graphify.helix.model import graphify_attributes + from graphify.helix.persistence import load_graph + + loaded = load_graph(graph_path) + except (OSError, RuntimeError, ValueError): return None - labels: dict[str, str] = {} - if labels_path.exists(): - try: - labels = json.loads(labels_path.read_text(encoding="utf-8")) - except (OSError, ValueError): - labels = {} - # id -> label from the graph, so a label-form citation resolves to a community too. - id_to_label: dict[str, str] = {} - try: - gdata = json.loads(graph_path.read_text(encoding="utf-8")) - for n in gdata.get("nodes", []): - if isinstance(n, dict) and n.get("id") is not None and n.get("label") is not None: - id_to_label[str(n["id"])] = str(n["label"]) - except (OSError, ValueError): - id_to_label = {} - # Sorted cid iteration + setdefault makes any label collision resolve - # deterministically (smallest community id wins). node_community: dict[str, str] = {} - for cid in sorted(communities, key=str): - label = labels.get(str(cid)) or labels.get(cid) or f"Community {cid}" - for nid in communities[cid]: - nid = str(nid) - node_community.setdefault(nid, label) - nlabel = id_to_label.get(nid) - if nlabel is not None: - node_community.setdefault(nlabel, label) + for record in loaded.state.get("communities", []): + if not isinstance(record, dict) or not isinstance(record.get("id"), int): + continue + label = str(record.get("name") or f"Community {record['id']}") + for node_id in record.get("members", []): + node_community.setdefault(str(node_id), label) + node = loaded.graph.node(node_id) + if node is not None: + attrs = graphify_attributes(node.attributes) + node_community.setdefault(str(attrs.get("label", node_id)), label) return node_community @@ -214,26 +187,24 @@ def _load_known_nodes(graph_path: Path) -> set[str] | None: Used to drop source nodes from lessons once the code they pointed at is gone (deleted/renamed) — a stale lesson shouldn't keep getting recommended. Both ids and labels are collected because `save-result` records source nodes by their - human-readable label (what an agent cites, e.g. ``build_from_json()``), while - graph nodes are keyed by id (e.g. ``module_build_from_json``). Matching on either + human-readable label (what an agent cites, e.g. ``build_from_extraction()``), while + graph nodes are keyed by id (e.g. ``module_build_from_extraction``). Matching on either keeps a still-present node and only drops one that survives under neither name — indexing ids alone silently dropped every label-form citation (the common case). """ try: - data = json.loads(Path(graph_path).read_text(encoding="utf-8")) - except (OSError, ValueError): - return None - nodes = data.get("nodes") - if not isinstance(nodes, list): + from graphify.helix.model import graphify_attributes + from graphify.helix.persistence import load_graph + + graph = load_graph(graph_path).graph + except (OSError, RuntimeError, ValueError): return None known: set[str] = set() - for n in nodes: - if not isinstance(n, dict): - continue - if n.get("id") is not None: - known.add(str(n["id"])) - if n.get("label") is not None: - known.add(str(n["label"])) + for node in graph.nodes(): + known.add(str(node.id)) + attrs = graphify_attributes(node.attributes) + if attrs.get("label") is not None: + known.add(str(attrs["label"])) return known or None @@ -525,11 +496,9 @@ def _topic_key(label: str) -> tuple[int, str]: def lessons_fresh(out_path: Path, memory_dir: Path, - graph_path: Path | None = None, - analysis_path: Path | None = None, - labels_path: Path | None = None) -> bool: + graph_path: Path | None = None) -> bool: """True if ``out_path`` exists and is at least as new as every input that - feeds it (the memory docs, and the graph/sidecars when one is used). + feeds it (the memory docs and native store when one is used). Lets ``graphify reflect --if-stale`` skip a redundant run — e.g. when the git post-commit hook just regenerated ``LESSONS.md`` and an agent then runs reflect @@ -550,7 +519,7 @@ def lessons_fresh(out_path: Path, memory_dir: Path, newest = max(newest, f.stat().st_mtime) except OSError: pass - for input_path in (graph_path, analysis_path, labels_path): + for input_path in (graph_path,): if input_path is None: continue gp = Path(input_path) @@ -563,8 +532,6 @@ def lessons_fresh(out_path: Path, memory_dir: Path, def reflect(memory_dir: Path, out_path: Path, graph_path: Path | None = None, - analysis_path: Path | None = None, - labels_path: Path | None = None, *, now: datetime | None = None, half_life_days: float = _DEFAULT_HALF_LIFE_DAYS, @@ -581,11 +548,7 @@ def reflect(memory_dir: Path, out_path: Path, known_nodes = None if graph_path is not None: graph_path = Path(graph_path) - analysis_path = Path(analysis_path) if analysis_path else ( - graph_path.parent / ".graphify_analysis.json") - labels_path = Path(labels_path) if labels_path else ( - graph_path.parent / ".graphify_labels.json") - node_community = _load_node_community(graph_path, analysis_path, labels_path) + node_community = _load_node_community(graph_path) known_nodes = _load_known_nodes(graph_path) if now is None: @@ -599,51 +562,43 @@ def reflect(memory_dir: Path, out_path: Path, out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text(render_lessons_md(agg), encoding="utf-8") - # Also project a derived experiential sidecar next to graph.json when a graph - # is in hand. Best-effort: a sidecar failure must never break LESSONS.md. + # Persist the experiential projection inside a new atomic Helix generation. if graph_path is not None: - try: - write_learning_sidecar(agg, Path(graph_path), now=now) - except Exception: - pass + write_learning_state(agg, Path(graph_path), now=now) return out_path, agg -# --- work-memory overlay sidecar ------------------------------------------------ -# -# A derived, experiential projection of the reflect aggregate, written next to -# graph.json as ``.graphify_learning.json``. It carries which nodes have proven -# preferred/tentative/contested, a code fingerprint for staleness detection, and -# a short provenance trail. graph.json (durable structural truth) is never -# touched — read surfaces merge this overlay in only at display time. +# --- work-memory overlay ------------------------------------------------------- def _build_id_label_maps(graph_path: Path) -> tuple[dict[str, str], dict[str, list[str]], dict[str, dict[str, Any]]]: - """From graph.json build: + """From an active Helix generation build: - ``id_set``: id -> id (every node id, so an id-form citation resolves to itself) - ``label_to_ids``: label -> [ids] (so a label-form citation can be resolved, and ambiguity — one label, many ids — can be detected and skipped) - ``node_by_id``: id -> node dict (for source_file lookup) - Best-effort; an unreadable/garbage graph yields empty maps. + Best-effort; an unreadable store yields empty maps. """ id_set: dict[str, str] = {} label_to_ids: dict[str, list[str]] = {} node_by_id: dict[str, dict[str, Any]] = {} try: - data = json.loads(Path(graph_path).read_text(encoding="utf-8")) - except (OSError, ValueError): + from graphify.helix.model import graphify_attributes + from graphify.helix.persistence import load_graph + + graph = load_graph(graph_path).graph + except (OSError, RuntimeError, ValueError): return id_set, label_to_ids, node_by_id - for n in data.get("nodes", []): - if not isinstance(n, dict) or n.get("id") is None: - continue - nid = str(n["id"]) + for node in graph.nodes(): + attrs = graphify_attributes(node.attributes) + nid = str(node.id) id_set[nid] = nid - node_by_id[nid] = n - label = n.get("label") + node_by_id[nid] = {"id": node.id, **attrs} + label = attrs.get("label") if label is not None: label_to_ids.setdefault(str(label), []).append(nid) return id_set, label_to_ids, node_by_id @@ -668,9 +623,8 @@ def _resolve_canonical_id(cited: str, id_set: dict[str, str], def _resolve_source_path(src: str, graph_path: Path) -> Path | None: """Locate a node's ``source_file`` on disk, returning an existing file or None. - ``source_file`` is stored relative to the PROJECT root, but graph.json may - live in ``/graphify-out/`` (so its own dir is not the root) or directly - at the root (``extract --out .``). Resolve the root in the most-likely order + ``source_file`` is stored relative to the project root, while graph.helix + normally lives in ``/graphify-out/``. Resolve roots in the most-likely order and return the first candidate where the file actually exists, so a defeated heuristic or a stale marker can never strand the file (every node would then look "changed"). The same search runs at write and read time, so the writer @@ -678,8 +632,8 @@ def _resolve_source_path(src: str, graph_path: Path) -> Path | None: Order: the committed ``.graphify_root`` marker (#686/#1423 — authoritative for an absolute/elsewhere ``GRAPHIFY_OUT`` override); then the layout-appropriate - root *first* — graph.json's parent's parent for the ``graphify-out`` layout, - or graph.json's own dir for a flat layout — which avoids matching a same-named + root first — graph.helix's parent's parent for the standard layout, + or the store's parent for a flat layout — which avoids matching a same-named file one directory up; then the other of the two; then the cwd. """ if not src: @@ -821,24 +775,23 @@ def _add(entry_src: dict[str, Any], status: str) -> None: } -def write_learning_sidecar(agg: dict[str, Any], graph_path: Path, - *, now: datetime | None = None) -> Path: - """Write ``.graphify_learning.json`` next to ``graph_path`` deterministically. +def write_learning_state(agg: dict[str, Any], graph_path: Path, + *, now: datetime | None = None) -> Path: + """Atomically persist the learning projection with native topology.""" + import copy + from graphify.helix.persistence import HelixEmbeddedStore - Sorted keys + indent=2 so re-runs on identical input (and a fixed ``now``) - are byte-identical. Returns the sidecar path. - """ overlay = build_learning_overlay(agg, graph_path, now=now) - sidecar = Path(graph_path).parent / LEARNING_SIDECAR_NAME - sidecar.write_text( - json.dumps(overlay, indent=2, sort_keys=True, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - return sidecar + with HelixEmbeddedStore(graph_path) as store: + previous_state = store.read_state() + state = copy.deepcopy(previous_state) + state["learning"] = overlay + store.replace_state(state, previous_state=previous_state) + return Path(graph_path) def load_learning_overlay(graph_path: Path) -> dict[str, dict[str, Any]]: - """Load the sidecar next to ``graph_path`` and return ``{node_id -> entry}`` + """Load native learning state and return ``{node_id -> entry}`` with a recomputed ``stale: bool`` per entry. Best-effort -> {} on any error. Staleness: recompute ``file_hash(source_file)`` and compare to the entry's @@ -847,10 +800,11 @@ def load_learning_overlay(graph_path: Path) -> dict[str, dict[str, Any]]: direction). An entry with no stored fingerprint AND no current file is not marked stale (nothing to re-verify). """ - sidecar = Path(graph_path).parent / LEARNING_SIDECAR_NAME try: - data = json.loads(sidecar.read_text(encoding="utf-8")) - except (OSError, ValueError): + from graphify.helix.persistence import load_graph + + data = dict(load_graph(graph_path).state.get("learning", {})) + except (OSError, RuntimeError, ValueError): return {} nodes = data.get("nodes") if not isinstance(nodes, dict): diff --git a/graphify/report.py b/graphify/report.py index 248bce9a1..d2d2d58c2 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -2,7 +2,9 @@ from __future__ import annotations import re from datetime import date -import networkx as nx +from typing import Any + +from .helix.model import edge_attributes, graphify_attributes, node_attributes def _safe_community_name(label: str) -> str: @@ -13,12 +15,10 @@ def _safe_community_name(label: str) -> str: def load_learning_for_report(graph_path) -> dict | None: - """Assemble the report's work-memory inputs from sibling artifacts. + """Assemble work-memory inputs from native state and memory documents. - Reads the ``.graphify_learning.json`` overlay (preferred sources) next to - ``graph_path`` and re-aggregates the memory docs for the query-scoped - dead-ends. Best-effort: returns None if neither is available, so the report - simply omits the section. Never raises. + Best-effort: returns None if neither is available, so the report simply + omits the section. Never raises. """ from pathlib import Path as _Path try: @@ -69,7 +69,7 @@ def _learning_section(lines: list, learning: dict | None, top_n: int = 10) -> No def generate( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], cohesion_scores: dict[int, float], community_labels: dict[int, str], @@ -90,13 +90,16 @@ def generate( if community_labels: community_labels = {int(k) if isinstance(k, str) else k: v for k, v in community_labels.items()} - confidences = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] + edge_rows = [ + (edge.source, edge.target, edge_attributes(edge)) for edge in G.edges() + ] + confidences = [data.get("confidence", "EXTRACTED") for _, _, data in edge_rows] total = len(confidences) or 1 ext_pct = round(confidences.count("EXTRACTED") / total * 100) inf_pct = round(confidences.count("INFERRED") / total * 100) amb_pct = round(confidences.count("AMBIGUOUS") / total * 100) - inf_edges = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "INFERRED"] + inf_edges = [(u, v, data) for u, v, data in edge_rows if data.get("confidence") == "INFERRED"] inf_scores = [d.get("confidence_score", 0.5) for _, _, d in inf_edges] inf_avg = round(sum(inf_scores) / len(inf_scores), 2) if inf_scores else None @@ -125,7 +128,7 @@ def generate( lines += [ "", "## Summary", - f"- {G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities" + f"- {G.node_count} nodes · {G.edge_count} edges · {len(communities)} communities" + (f" ({shown_count} shown, {thin_count_summary} thin omitted)" if thin_count_summary else ""), f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS" + (f" · INFERRED: {len(inf_edges)} edges (avg confidence: {inf_avg})" if inf_avg is not None else ""), @@ -189,10 +192,11 @@ def generate( # noise there ("None detected" on every run). Emit it only when the graph # actually contains code (#1657). _has_code = any( - d.get("file_type") == "code" for _, d in G.nodes(data=True) + graphify_attributes(node.attributes).get("file_type") == "code" + for node in G.nodes() ) or any( d.get("relation") in ("imports", "imports_from") - for *_e, d in G.edges(data=True) + for *_e, d in edge_rows ) if _has_code: from .analyze import find_import_cycles @@ -209,7 +213,8 @@ def generate( else: lines.append("- None detected.") - hyperedges = G.graph.get("hyperedges", []) + metadata = dict(G.attributes).get("graph", {}) + hyperedges = metadata.get("hyperedges", []) if isinstance(metadata, dict) else [] if hyperedges: lines += ["", "## Hyperedges (group relationships)"] for h in hyperedges: @@ -229,7 +234,7 @@ def generate( continue if len(real_nodes) < min_community_size: continue - display = [G.nodes[n].get("label", n) for n in real_nodes[:8]] + display = [node_attributes(G, n).get("label", n) for n in real_nodes[:8]] suffix = f" (+{len(real_nodes)-8} more)" if len(real_nodes) > 8 else "" lines += [ "", @@ -238,12 +243,12 @@ def generate( f"Nodes ({len(real_nodes)}): {', '.join(display)}{suffix}", ] - ambiguous = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "AMBIGUOUS"] + ambiguous = [(u, v, data) for u, v, data in edge_rows if data.get("confidence") == "AMBIGUOUS"] if ambiguous: lines += ["", "## Ambiguous Edges - Review These"] for u, v, d in ambiguous: - ul = G.nodes[u].get("label", u) - vl = G.nodes[v].get("label", v) + ul = node_attributes(G, u).get("label", u) + vl = node_attributes(G, v).get("label", v) lines += [ f"- `{ul}` → `{vl}` [AMBIGUOUS]", f" {d.get('source_file', '')} · relation: {d.get('relation', 'unknown')}", @@ -253,11 +258,11 @@ def generate( from .analyze import _is_file_node, _is_concept_node isolated = [ - n for n in G.nodes() - if G.degree(n) <= 1 - and not _is_file_node(G, n) - and not _is_concept_node(G, n) - and G.nodes[n].get("file_type") != "rationale" + node.id for node in G.nodes() + if G.degree(node.id).degree <= 1 + and not _is_file_node(G, node.id) + and not _is_concept_node(G, node.id) + and graphify_attributes(node.attributes).get("file_type") != "rationale" ] thin_communities = { cid: nodes for cid, nodes in communities.items() @@ -268,7 +273,7 @@ def generate( if gap_count > 0 or amb_pct > 20: lines += ["", "## Knowledge Gaps"] if isolated: - isolated_labels = [G.nodes[n].get("label", n) for n in isolated[:5]] + isolated_labels = [node_attributes(G, n).get("label", n) for n in isolated[:5]] suffix = f" (+{len(isolated)-5} more)" if len(isolated) > 5 else "" lines.append(f"- **{len(isolated)} isolated node(s):** {', '.join(f'`{l}`' for l in isolated_labels)}{suffix}") lines.append(" These have ≤1 connection - possible missing edges or undocumented components.") @@ -278,7 +283,7 @@ def generate( lines.append(f"- **High ambiguity: {amb_pct}% of edges are AMBIGUOUS.** Review the Ambiguous Edges section above.") # --- Work-memory lessons (derived overlay) --- - # Preferred sources come from the .graphify_learning.json sidecar; the + # Preferred sources come from native learning state; the # query-scoped dead-ends come from the reflect aggregate. Section omitted # entirely when neither is present, so a graph with no work-memory is # byte-identical to the pre-feature report. diff --git a/graphify/scip_ingest.py b/graphify/scip_ingest.py index bf3d1857c..04f887de0 100644 --- a/graphify/scip_ingest.py +++ b/graphify/scip_ingest.py @@ -14,7 +14,7 @@ extraction result format. All edges emitted are endpoint-safe — the function builds a symbol → node_id index in a first pass and either resolves relationship targets via that index or creates a stub - external node so `build_from_json()` will keep the edge. + external node so `build_from_extraction()` will keep the edge. Supported (simplified) JSON shape: documents[]: { relative_path, language, symbols[] } diff --git a/graphify/security.py b/graphify/security.py index 2dbe5bd77..0bf73ca67 100644 --- a/graphify/security.py +++ b/graphify/security.py @@ -21,50 +21,6 @@ _MAX_FETCH_BYTES = 52_428_800 # 50 MB hard cap for binary downloads _MAX_TEXT_BYTES = 10_485_760 # 10 MB hard cap for HTML / text -# Graph-load memory-bomb cap: reject .json files larger than this before -# JSON-parsing them into a dict. Without this, a multi-gigabyte (or -# specifically crafted) graph.json can exhaust process memory during -# json.loads + node_link_graph rehydration. -# Default fallback cap. Kept as a module-level constant so the value is -# discoverable and so existing callers/tests that reference it directly keep -# working; the effective cap is resolved at call time by -# ``_max_graph_file_bytes`` (which lets ``GRAPHIFY_MAX_GRAPH_BYTES`` override it). -_MAX_GRAPH_FILE_BYTES = 512 * 1024 * 1024 # 512 MiB - - -def _max_graph_file_bytes() -> int: - """Return the graph.json size cap in bytes. - - Honors the ``GRAPHIFY_MAX_GRAPH_BYTES`` environment variable so users with - large codebases can raise the limit without editing source. The value may - be plain bytes (``671088640``) or carry an ``MB`` / ``GB`` suffix - (``640MB``, ``2GB`` — case-insensitive, binary multipliers: ``MB`` is - 1024*1024 and ``GB`` is 1024*1024*1024, i.e. MiB / GiB). - Falls back to ``_MAX_GRAPH_FILE_BYTES`` (512 MiB) when the env var is unset, - blank, or unparseable. - - Read fresh on every call so the env var can be set before import and still - take effect. - """ - raw = os.environ.get("GRAPHIFY_MAX_GRAPH_BYTES", "").strip() - if not raw: - return _MAX_GRAPH_FILE_BYTES - text = raw.upper() - multiplier = 1 - if text.endswith("GB"): - multiplier = 1024 * 1024 * 1024 - text = text[:-2].strip() - elif text.endswith("MB"): - multiplier = 1024 * 1024 - text = text[:-2].strip() - try: - value = int(text) - except ValueError: - return _MAX_GRAPH_FILE_BYTES - if value <= 0: - return _MAX_GRAPH_FILE_BYTES - return value * multiplier - # AWS metadata, link-local, and common cloud metadata endpoints _BLOCKED_HOSTS = {"metadata.google.internal", "metadata.google.com"} @@ -308,81 +264,6 @@ def safe_fetch_text(url: str, max_bytes: int = _MAX_TEXT_BYTES, timeout: int = 1 return raw.decode("utf-8", errors="replace") -# --------------------------------------------------------------------------- -# Path validation -# --------------------------------------------------------------------------- - -def validate_graph_path(path: str | Path, base: Path | None = None) -> Path: - """Resolve *path* and verify it stays inside *base*. - - *base* defaults to the `graphify-out` directory relative to CWD. - Also requires the base directory to exist, so a caller cannot - trick graphify into reading files before any graph has been built. - - Raises: - ValueError - path escapes base, or base does not exist - FileNotFoundError - resolved path does not exist - """ - if base is None: - resolved_hint = Path(path).resolve() - for candidate in [resolved_hint, *resolved_hint.parents]: - if candidate.name == GRAPHIFY_OUT_NAME: - base = candidate - break - if base is None: - base = Path(GRAPHIFY_OUT).resolve() - - base = base.resolve() - if not base.exists(): - raise ValueError( - f"Graph base directory does not exist: {base}. " - "Run /graphify first to build the graph." - ) - - resolved = Path(path).resolve() - try: - resolved.relative_to(base) - except ValueError: - raise ValueError( - f"Path {path!r} escapes the allowed directory {base}. " - "Only paths inside graphify-out/ are permitted." - ) - - if not resolved.exists(): - raise FileNotFoundError(f"Graph file not found: {resolved}") - - return resolved - - -def check_graph_file_size_cap(path: Path) -> None: - """Reject *path* if its size exceeds the configured graph-file cap. - - Protects callers from memory bombs by failing fast before a multi-GiB - graph.json is read into memory and JSON-parsed. Silently returns when - ``path.stat()`` cannot be read — the caller's own existence/path check - is expected to surface a clearer error in that case. - - The cap is resolved on every call via :func:`_max_graph_file_bytes`, so the - ``GRAPHIFY_MAX_GRAPH_BYTES`` env var can be set before import and still - apply. - - Raises: - ValueError - file size exceeds the cap. The message includes the - observed size, the cap, and how to raise the limit. - """ - cap = _max_graph_file_bytes() - try: - size = path.stat().st_size - except OSError: - return - if size > cap: - raise ValueError( - f"graph file {path} is {size:_d} bytes, exceeds {cap:_d}-byte cap\n" - f"(set GRAPHIFY_MAX_GRAPH_BYTES= or " - f"GRAPHIFY_MAX_GRAPH_BYTES=GB to raise the limit)" - ) - - # --------------------------------------------------------------------------- # Label sanitisation (mirrors code-review-graph's _sanitize_name pattern) # --------------------------------------------------------------------------- @@ -405,6 +286,18 @@ def sanitize_label(text: str | None) -> str: return text +def validate_store_path(path: str | Path) -> Path: + """Resolve and validate an embedded Helix store directory.""" + resolved = Path(path).expanduser().resolve() + if resolved.suffix.lower() == ".json" or resolved.is_file(): + raise ValueError( + "legacy JSON graphs are obsolete; pass a graph.helix store and rebuild from source" + ) + if not resolved.is_dir(): + raise FileNotFoundError(f"Helix store not found: {resolved}") + return resolved + + # --------------------------------------------------------------------------- # Metadata sanitisation (recursive, bounded, HTML-safe) # --------------------------------------------------------------------------- diff --git a/graphify/semantic_cleanup.py b/graphify/semantic_cleanup.py index 09cac2d84..30963413d 100644 --- a/graphify/semantic_cleanup.py +++ b/graphify/semantic_cleanup.py @@ -5,7 +5,7 @@ # `graphify merge-chunks` command — both ingest untrusted agent-written chunk # JSON, and validate_semantic_fragment() rejects malformed/oversized payloads and # crafted node/edge IDs before they touch the graph. The primary build/load paths -# (build_from_json, load_graph_json) deliberately do NOT run this: they must keep +# Build DTO assembly deliberately does not run this: it must keep # loading valid pre-existing graphs whose AST node IDs predate the stricter # semantic-ID charset. from __future__ import annotations @@ -83,7 +83,7 @@ def validate_semantic_fragment(fragment: object) -> list[str]: _validate_semantic_id(errors, f"nodes[{i}].id", node.get("id")) # file_type is intentionally NOT rejected here. It carries no security # risk (it can't exhaust memory or escape a directory), and - # build_from_json already coerces every value via _FILE_TYPE_SYNONYMS + # build_from_extraction already coerces every value via _FILE_TYPE_SYNONYMS # (unknown -> "concept", #840). Rejecting a whole chunk over a synonym # like "markdown"/"tool"/"framework" that the loader would happily map is # pure data loss, so leave file_type normalization to build. diff --git a/graphify/serve.py b/graphify/serve.py index 0a0e7bfe3..2596c099a 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -4,14 +4,12 @@ import math import re import sys -from array import array from pathlib import Path -from typing import NamedTuple -import networkx as nx -from networkx.readwrite import json_graph -from graphify.security import sanitize_label, check_graph_file_size_cap +from typing import Any, NamedTuple +from graphify.security import sanitize_label, validate_store_path from graphify.build import edge_data -from graphify.paths import default_graph_json as _default_graph_json +from graphify.helix.model import LoadedGraph, edge_attributes, graphify_attributes, node_attributes +from graphify.helix.persistence import DEFAULT_PROJECT_STORE, HelixGraphReader, load_graph try: import jieba as _jieba # type: ignore[import-untyped] @@ -19,57 +17,23 @@ _jieba = None -def _load_graph(graph_path: str) -> nx.Graph: +def _load_graph(graph_path: str) -> LoadedGraph: try: - resolved = Path(graph_path).resolve() - if resolved.suffix != ".json": - raise ValueError(f"Graph path must be a .json file, got: {graph_path!r}") - if not resolved.exists(): - raise FileNotFoundError(f"Graph file not found: {resolved}") - check_graph_file_size_cap(resolved) - safe = resolved - data = json.loads(safe.read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - data = {**data, "directed": True} - try: - from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(data.get("nodes", [])): - print( - "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " - "rebuild with `graphify extract --force` for path-qualified IDs.", - file=sys.stderr, - ) - except Exception: - pass - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) - # Attach the work-memory overlay (derived sidecar next to graph.json) so - # the query/MCP read surface can annotate NODE lines display-only. Empty - # when no sidecar exists, leaving un-annotated output byte-identical. - try: - from graphify.reflect import load_learning_overlay as _llo - G.graph["_learning_overlay"] = _llo(resolved) - except Exception: - G.graph["_learning_overlay"] = {} - return G + return load_graph(validate_store_path(graph_path)) except (ValueError, FileNotFoundError) as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1) - except json.JSONDecodeError as exc: - print(f"error: graph.json is corrupted ({exc}). Re-run /graphify to rebuild.", file=sys.stderr) + except RuntimeError as exc: + print(f"error: Helix store is corrupted ({exc}). Re-run /graphify to rebuild.", file=sys.stderr) sys.exit(1) -def _communities_from_graph(G: nx.Graph) -> dict[int, list[str]]: - """Reconstruct community dict from community property stored on nodes.""" +def _communities_from_graph(G: LoadedGraph) -> dict[int, list[str]]: + """Read communities from the same durable Helix generation.""" communities: dict[int, list[str]] = {} - for node_id, data in G.nodes(data=True): - cid = data.get("community") - if cid is not None: - communities.setdefault(int(cid), []).append(node_id) + for record in G.state.get("communities", []): + if isinstance(record, dict) and isinstance(record.get("id"), int): + communities[record["id"]] = list(record.get("members", [])) return communities @@ -190,135 +154,14 @@ def _query_terms(question: str) -> list[str]: _SOURCE_MATCH_BONUS = 0.5 -def _compute_idf(G: nx.Graph, terms: list[str]) -> dict[str, float]: - """IDF weights for query terms, cached in G.graph['_idf_cache']. - - Common terms like 'error' or 'exception' that match hundreds of nodes get - low weights; rare identifiers like 'FooBarService' get high weights. - Cache is stored on the graph object itself so it auto-invalidates when - a hot-reload replaces G with a new object. - """ - cache: dict[str, float] = G.graph.setdefault("_idf_cache", {}) - N = G.number_of_nodes() or 1 - uncached = [t for t in terms if t not in cache] - if uncached: - df: dict[str, int] = {t: 0 for t in uncached} - for _, data in G.nodes(data=True): - norm_label = ( - data.get("norm_label") or _strip_diacritics(data.get("label") or "") - ).lower() - for t in uncached: - if t in norm_label: - df[t] += 1 - for t in uncached: - cache[t] = math.log(1 + N / (1 + df[t])) - return {t: cache.get(t, math.log(1 + N)) for t in terms} - - -def _trigrams(text: str) -> set[str]: - """Character trigrams of `text`; for <3-char text the whole string is the key.""" - if len(text) < 3: - return {text} if text else set() - return {text[i:i + 3] for i in range(len(text) - 2)} - - -def _node_search_text(data: dict, nid: str) -> str: - """Concatenate every field _score_nodes / _find_node match a query against, so - one trigram index over this text is a complete candidate generator for both. - - - `norm_label` and `source_file` feed _score_nodes' per-term substring tiers. - - `label_tokens` (the space-joined token form) feeds _find_node's - `term in label_tokens` branch, where a multi-word `term` can span a token - boundary that punctuation hides in `norm_label` (e.g. query "foo bar" matches - label "foo.bar" only via its tokenized form). - - `source_tokens` feeds _find_node's exact source-file path lookup, where a - query like "app/api/example/route.ts" tokenizes to "app api example route ts". - - `nid` feeds the whole-query `joined == nid_lower` tier. - - NUL separators stop a trigram from spanning two fields (a query never contains - NUL, so a cross-field trigram can never be a real match). - """ - norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower() - label_tokens = " ".join(_search_tokens(data.get("label") or "")) - source = (data.get("source_file") or "").lower() - source_tokens = " ".join(_search_tokens(data.get("source_file") or "")) - return "\x00".join((norm_label, label_tokens, str(nid).lower(), source, source_tokens)) - - -def _get_trigram_index(G: nx.Graph) -> dict: - """Lazily build and cache a trigram -> node-position postings map on the graph. - - Cached on `G.graph` so it auto-invalidates when a hot-reload swaps in a - fresh graph object, exactly like `_idf_cache`. `set_cache` memoizes per-trigram - id-sets across queries within one graph generation. - """ - idx = G.graph.get("_trigram_index") - if idx is not None: - return idx - ids = list(G.nodes()) - postings: dict[str, array] = {} - for i, nid in enumerate(ids): - for g in _trigrams(_node_search_text(G.nodes[nid], nid)): - bucket = postings.get(g) - if bucket is None: - bucket = array("i") - postings[g] = bucket - bucket.append(i) - idx = {"ids": ids, "postings": postings, "set_cache": {}} - G.graph["_trigram_index"] = idx - return idx - - -def _trigram_candidates(G: nx.Graph, needles: list[str], *, guard_frac: float = 0.10) -> list[str] | None: - """Node IDs whose text could contain any `needle` as a substring, via the - trigram index — a *superset* the caller then re-scores with the exact predicates. - - Returns candidates in graph-iteration order (so order-sensitive callers like - _find_node stay byte-identical to a full scan), or **None** when the index isn't - worth it — a needle is too short to trigram, or its rarest trigram is still - common enough that the candidate set would approach the whole graph. The caller - falls back to the full scan, preserving the never-worse contract. The guard is - cheap: postings-length lookups only, no set intersection. - """ - idx = _get_trigram_index(G) - ids, postings, set_cache = idx["ids"], idx["postings"], idx["set_cache"] - n = len(ids) - if n == 0: - return [] - needles = [s for s in needles if s] - thresh = int(n * guard_frac) - for s in needles: - tgs = _trigrams(s) - if not tgs or any(len(g) < 3 for g in tgs): - return None # too short to trigram-filter - present = [len(postings[g]) for g in tgs if g in postings] - if not present: - continue # this needle matches nothing — contributes no candidates - if min(present) > thresh: - return None # rarest trigram still too common -> not worth the index - cand: set[int] = set() - for s in needles: - sets: list[set] | None = [] - for g in _trigrams(s): - bucket = postings.get(g) - if bucket is None: - sets = None # a trigram absent everywhere -> needle matches nothing - break - cached = set_cache.get(g) - if cached is None: - cached = set(bucket) - set_cache[g] = cached - sets.append(cached) - if not sets: - continue - sets.sort(key=len) # intersect smallest-first - hit = set(sets[0]) - for other in sets[1:]: - hit &= other - if not hit: - break - cand |= hit - return [ids[i] for i in sorted(cand)] +def _compute_idf(G: Any, terms: list[str], native_query: Any) -> dict[str, float]: + """Compute exact label document frequencies with public Helix predicates.""" + frequencies = native_query.document_frequencies(terms) + node_count = G.node_count or 1 + return { + term: math.log(1 + node_count / (1 + frequencies.get(term, 0))) + for term in terms + } class _QueryScores(NamedTuple): @@ -337,7 +180,9 @@ class _QueryScores(NamedTuple): best_seed_by_term: dict[str, str] -def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: +def _score_nodes( + G: Any, terms: list[str], *, native_query: Any +) -> list[tuple[float, str]]: """Combined query scorer returning the existing ranked `(score, node_id)` list. Backwards-compatible thin wrapper around `_score_query` for path, explain, @@ -345,11 +190,17 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: per-term seed metadata computed by `_score_query` (when requested) is discarded here so existing callers see no API or runtime-cost change. """ - return _score_query(G, terms, collect_per_term_seeds=False).ranked + return _score_query( + G, terms, collect_per_term_seeds=False, native_query=native_query + ).ranked def _score_query( - G: nx.Graph, terms: list[str], *, collect_per_term_seeds: bool + G: Any, + terms: list[str], + *, + collect_per_term_seeds: bool, + native_query: Any, ) -> _QueryScores: """Single-pass combined scorer that optionally also records the best seed for each normalized query token. @@ -383,21 +234,16 @@ def _score_query( # below it would also inflate the matched-term ratio (#1602). norm_terms = list(dict.fromkeys(tok for t in terms for tok in _search_tokens(t))) n_terms = len(norm_terms) - idf = _compute_idf(G, norm_terms) + idf = _compute_idf(G, norm_terms, native_query) # Whole-query string for full-label matching (mirrors _find_node's `term`). joined = " ".join(norm_terms) # Weight the full-query bonus by the rarest constituent term so a specific # multi-word label still outweighs common-token noise; floor at 1.0. joined_w = max((idf.get(t, 1.0) for t in norm_terms), default=1.0) - # Trigram prefilter: score only nodes whose text could match a term, falling - # back to the whole graph when the index isn't selective. The result is - # identical either way — the per-node scoring below is unchanged and a - # non-candidate node always scores 0. (IDF above stays a whole-graph statistic.) - candidate_ids = _trigram_candidates(G, norm_terms + ([joined] if joined else [])) - node_iter = ( - G.nodes(data=True) if candidate_ids is None - else ((nid, G.nodes[nid]) for nid in candidate_ids) + candidate_ids = native_query.candidate_ids( + norm_terms + ([joined] if joined else []) ) + node_iter = ((nid, node_attributes(G, nid)) for nid in candidate_ids) # Per-token best tracking, only when the caller (the query path) wants the # seed metadata. The key tuple is the full multi-key tie-break # (`(-singleton_score, -degree, label_len, nid)`), so `min` over the @@ -422,7 +268,7 @@ def _score_query( # the per-token singleton tier (joined-singlet exact-match check). When # neither runs (`joined` empty AND not collecting seeds) skip the call; # this preserves the single-query-time perf where nid_lower was lazy. - nid_lower = nid.lower() if (joined or collect_per_term_seeds) else "" + nid_lower = str(nid).lower() if (joined or collect_per_term_seeds) else "" score = 0.0 # Full-query tier: a multi-word query that equals (or prefixes) the whole # label must dominate the per-token bag-of-words sums below, so `path`/ @@ -501,7 +347,7 @@ def _score_query( # (-singleton, -degree, label_len, nid) — the minimum # tuple wins, exactly matching max(tied, key=degree) # over (label_len asc, nid asc)-sorted ties. - key = (-singleton, -G.degree(nid), len(data.get("label") or nid), nid) + key = (-singleton, -G.degree(nid).degree, len(data.get("label") or str(nid)), str(nid)) cur = best_by_term.get(t) if cur is None or key < cur[0]: best_by_term[t] = (key, nid) @@ -511,14 +357,14 @@ def _score_query( scored.append((score, nid)) # Sort by score desc; break ties toward the shorter label so a concise exact # match beats a longer superset that happens to share the same score. - scored.sort(key=lambda s: (-s[0], len(G.nodes[s[1]].get("label") or s[1]), s[1])) + scored.sort(key=lambda s: (-s[0], len(node_attributes(G, s[1]).get("label") or str(s[1])), str(s[1]))) best_seed_by_term: dict[str, str] = {} if collect_per_term_seeds and best_by_term: best_seed_by_term = {t: nid for t, (_key, nid) in best_by_term.items()} return _QueryScores(ranked=scored, best_seed_by_term=best_seed_by_term) -def _pick_scored_endpoint(G: nx.Graph, scored: list[tuple[float, str]], query: str) -> str: +def _pick_scored_endpoint(G: Any, scored: list[tuple[float, str]], query: str) -> str: """Pick a path endpoint from a _score_nodes result, preferring full-token matches. The full-query tier in _score_nodes only fires when the query equals or @@ -537,7 +383,7 @@ def _pick_scored_endpoint(G: nx.Graph, scored: list[tuple[float, str]], query: s if not qtokens: return scored[0][1] for _score, nid in scored: - if qtokens <= set(_search_tokens(G.nodes[nid].get("label") or nid)): + if qtokens <= set(_search_tokens(node_attributes(G, nid).get("label") or str(nid))): return nid return scored[0][1] @@ -547,7 +393,7 @@ def _pick_seeds( max_k: int = 3, gap_ratio: float = 0.2, *, - G: "nx.Graph | None" = None, + G: Any | None = None, best_seed_by_term: dict[str, str] | None = None, ) -> list[str]: """Select BFS seed nodes, stopping when score drops too far below the top. @@ -593,7 +439,7 @@ def _pick_seeds( def _seed_label_key(nid: str) -> str: if G is None: return nid - data = G.nodes[nid] + data = node_attributes(G, nid) return (data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower()) or nid @@ -720,80 +566,113 @@ def _resolve_context_filters(question: str, explicit_filters: list[str] | None = return [], None -def _filter_graph_by_context(G: nx.Graph, context_filters: list[str] | None) -> nx.Graph: - filters = set(_normalize_context_filters(context_filters)) - if not filters: - return G - H = G.__class__() - H.add_nodes_from(G.nodes(data=True)) - if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): - for u, v, key, data in G.edges(keys=True, data=True): - if data.get("context") in filters: - H.add_edge(u, v, key=key, **data) - else: - for u, v, data in G.edges(data=True): - if data.get("context") in filters: - H.add_edge(u, v, **data) - return H +def _filter_graph_by_context(G: Any, context_filters: list[str] | None) -> tuple[Any, set[str]]: + """Return the native snapshot plus normalized traversal edge filters.""" + return G, set(_normalize_context_filters(context_filters)) -def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: +def _traverse( + G: Any, + start_nodes: list[str], + depth: int, + *, + strategy: str, + context_filters: set[str] | None = None, + native_query: Any | None = None, +) -> tuple[set[str], list[Any]]: # Compute hub threshold: nodes above this degree are not expanded as transit. # p99 of degree distribution, floored at 50 to avoid over-blocking small graphs. - degrees = [G.degree(n) for n in G.nodes()] + degrees = [int(row.degree) for row in G.degrees()] if degrees: degrees_sorted = sorted(degrees) p99_idx = int(len(degrees_sorted) * 0.99) hub_threshold = max(50, degrees_sorted[p99_idx]) else: hub_threshold = 50 - seed_set = set(start_nodes) - visited: set[str] = set(start_nodes) - frontier = set(start_nodes) - edges_seen: list[tuple] = [] - for _ in range(depth): - next_frontier: set[str] = set() - for n in frontier: - # Don't expand through high-degree hubs (except seeds - a hub that - # is the starting node should still be explored). - if n not in seed_set and G.degree(n) >= hub_threshold: + if not context_filters: + from helixdb import TraversalOptions + + result = G.traverse(TraversalOptions( + seeds=tuple(start_nodes), + max_depth=depth, + strategy=strategy, + direction="both", + stop_non_seed_at_or_above_degree=hub_threshold, + )) + return ( + {visit.node_id for visit in result.visits}, + [G.edge(item.edge_id) for item in result.discovery_edges if G.edge(item.edge_id) is not None], + ) + + if native_query is None: + raise RuntimeError("context-filtered traversal requires a native Helix query") + visited = set(native_query.traverse_ids( + start_nodes, depth, contexts=context_filters + )) + edges_seen: list[Any] = [] + seen_edge_ids: set[Any] = set() + for node in visited: + for edge_id in G.incident_edge_ids(node): + if edge_id in seen_edge_ids: continue - for neighbor in G.neighbors(n): - if neighbor not in visited: - next_frontier.add(neighbor) - edges_seen.append((n, neighbor)) - visited.update(next_frontier) - frontier = next_frontier + edge = G.edge(edge_id) + if ( + edge is None + or edge.source not in visited + or edge.target not in visited + or edge_attributes(edge).get("context") not in context_filters + ): + continue + seen_edge_ids.add(edge_id) + edges_seen.append(edge) return visited, edges_seen -def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: - degrees = [G.degree(n) for n in G.nodes()] - if degrees: - degrees_sorted = sorted(degrees) - p99_idx = int(len(degrees_sorted) * 0.99) - hub_threshold = max(50, degrees_sorted[p99_idx]) - else: - hub_threshold = 50 - seed_set = set(start_nodes) - visited: set[str] = set() - edges_seen: list[tuple] = [] - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, d = stack.pop() - if node in visited or d > depth: - continue - visited.add(node) - if node not in seed_set and G.degree(node) >= hub_threshold: - continue - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, d + 1)) - edges_seen.append((node, neighbor)) - return visited, edges_seen +def _bfs( + G: Any, + start_nodes: list[str], + depth: int, + context_filters: set[str] | None = None, + *, + native_query: Any | None = None, +): + return _traverse( + G, + start_nodes, + depth, + strategy="breadth_first", + context_filters=context_filters, + native_query=native_query, + ) -def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_budget: int = 2000, *, seeds: list[str] | None = None) -> str: +def _dfs( + G: Any, + start_nodes: list[str], + depth: int, + context_filters: set[str] | None = None, + *, + native_query: Any | None = None, +): + return _traverse( + G, + start_nodes, + depth, + strategy="depth_first", + context_filters=context_filters, + native_query=native_query, + ) + + +def _subgraph_to_text( + G: Any, + nodes: set[str], + edges: list[Any], + token_budget: int = 2000, + *, + seeds: list[str] | None = None, + learning_overlay: dict[str, Any] | None = None, +) -> str: """Render subgraph as text, cutting at token_budget (approx 3 chars/token). seeds: exact-match nodes rendered first before the degree-sorted expansion, @@ -801,14 +680,12 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu """ char_budget = token_budget * 3 lines = [] - # Work-memory overlay (derived sidecar) stashed on the graph at load time. - # Empty when no sidecar exists, so un-annotated output stays byte-identical. - overlay = getattr(G, "graph", {}).get("_learning_overlay", {}) or {} + overlay = learning_overlay or {} seed_set = set(seeds or []) ordered = [n for n in (seeds or []) if n in nodes] + \ - sorted(nodes - seed_set, key=lambda n: G.degree(n), reverse=True) + sorted(nodes - seed_set, key=lambda n: G.degree(n).degree, reverse=True) for nid in ordered: - d = G.nodes[nid] + d = node_attributes(G, nid) # Every LLM-derived field passes through sanitize_label before being # concatenated into MCP tool output (F-010): an attacker who controls a # corpus document can otherwise inject ANSI escapes, fake graphify-out @@ -830,17 +707,17 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu f"{learning_suffix}]" ) lines.append(line) - for u, v in edges: + for traversed in edges: + u, v = traversed.source, traversed.target if u in nodes and v in nodes: - raw = G[u][v] - d = next(iter(raw.values()), {}) if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)) else raw + d = edge_attributes(traversed) context = d.get("context") context_suffix = f" context={sanitize_label(str(context))}" if context else "" line = ( - f"EDGE {sanitize_label(G.nodes[u].get('label', u))} " + f"EDGE {sanitize_label(node_attributes(G, u).get('label', u))} " f"--{sanitize_label(str(d.get('relation', '')))} " f"[{sanitize_label(str(d.get('confidence', '')))}{context_suffix}]--> " - f"{sanitize_label(G.nodes[v].get('label', v))}" + f"{sanitize_label(node_attributes(G, v).get('label', v))}" ) lines.append(line) output = "\n".join(lines) @@ -859,13 +736,15 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu def _query_graph_text( - G: nx.Graph, + G: Any, question: str, *, + native_query: Any, mode: str = "bfs", depth: int = 3, token_budget: int = 2000, context_filters: list[str] | None = None, + learning_overlay: dict[str, Any] | None = None, ) -> str: terms = _query_terms(question) # One graph scoring pass produces both the combined ranking (used to drive @@ -874,25 +753,52 @@ def _query_graph_text( # — one combined + one per query token — re-walking the whole graph each # time; on a 100k-node, three-term benchmark ~71% of scoring time was # spent in those redundant per-term passes. - qs = _score_query(G, terms, collect_per_term_seeds=True) + qs = _score_query( + G, + terms, + collect_per_term_seeds=True, + native_query=native_query, + ) start_nodes = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) if not start_nodes: return "No matching nodes found." resolved_filters, filter_source = _resolve_context_filters(question, context_filters) - traversal_graph = _filter_graph_by_context(G, resolved_filters) - nodes, edges = _dfs(traversal_graph, start_nodes, depth) if mode == "dfs" else _bfs(traversal_graph, start_nodes, depth) + traversal_graph, edge_filters = _filter_graph_by_context(G, resolved_filters) + nodes, edges = ( + _dfs( + traversal_graph, + start_nodes, + depth, + edge_filters, + native_query=native_query, + ) + if mode == "dfs" + else _bfs( + traversal_graph, + start_nodes, + depth, + edge_filters, + native_query=native_query, + ) + ) header_parts = [ f"Traversal: {mode.upper()} depth={depth}", - f"Start: {[G.nodes[n].get('label', n) for n in start_nodes]}", + f"Start: {[node_attributes(G, n).get('label', n) for n in start_nodes]}", ] if resolved_filters: header_parts.append(f"Context: {', '.join(resolved_filters)} ({filter_source})") header_parts.append(f"{len(nodes)} nodes found") header = " | ".join(header_parts) + "\n\n" - return header + _subgraph_to_text(traversal_graph, nodes, edges, token_budget) + return header + _subgraph_to_text( + traversal_graph, + nodes, + edges, + token_budget, + learning_overlay=learning_overlay, + ) -def _find_node(G: nx.Graph, label: str) -> list[str]: +def _find_node(G: Any, label: str, *, native_query: Any) -> list[str]: """Return node IDs whose label or ID matches the search term (diacritic-insensitive). Results are ordered by precedence: exact source-file path match first, then @@ -913,19 +819,14 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: exact: list[str] = [] prefix: list[str] = [] substring: list[str] = [] - # Trigram prefilter (graph-iteration order preserved so exact/prefix/substring - # ordering — and thus matches[0] — is byte-identical to the full scan). - candidate_ids = _trigram_candidates(G, [term, norm_query]) - node_iter = ( - G.nodes(data=True) if candidate_ids is None - else ((nid, G.nodes[nid]) for nid in candidate_ids) - ) + candidate_ids = native_query.candidate_ids([term, norm_query]) + node_iter = ((nid, node_attributes(G, nid)) for nid in candidate_ids) for nid, d in node_iter: norm_label = d.get("norm_label") or _strip_diacritics(d.get("label") or "").lower() bare_label = norm_label.rstrip("()") label_tokens = " ".join(_search_tokens(d.get("label") or "")) source_tokens = " ".join(_search_tokens(d.get("source_file") or "")) - nid_lower = nid.lower() + nid_lower = str(nid).lower() if term == source_tokens: source_exact.append(nid) elif ( @@ -950,8 +851,8 @@ def _find_node(G: nx.Graph, label: str) -> list[str]: preferred = [ nid for nid in source_exact - if str(G.nodes[nid].get("source_location", "")) == "L1" - and _strip_diacritics(str(G.nodes[nid].get("label") or "")).lower() + if str(node_attributes(G, nid).get("source_location", "")) == "L1" + and _strip_diacritics(str(node_attributes(G, nid).get("label") or "")).lower() == query_basename ] if len(preferred) == 1: @@ -992,7 +893,7 @@ def _relay() -> None: def _community_header(cid: int, community_name) -> str: # Header for get_community: "Community N — Name", matching get_node / query - # output which read the community_name attribute to_json writes onto nodes. + # output which reads community names from native generation state. # Skip the name when it is just the "Community N" placeholder (written for # unnamed communities) so the header never reads "Community 12 — Community 12"; # also falls back to the bare id when there is no name. Name is sanitised @@ -1010,7 +911,7 @@ def _build_server(graph_path: str): All graph query tools and resources are registered here over a single ``mcp.server.Server`` instance; the caller picks the transport (stdio or - Streamable HTTP) and runs it. Hot-reload of graph.json works the same way + Streamable HTTP) and runs it. Helix generation hot reload works the same way regardless of transport, since reloads happen inside the tool handlers. """ import threading @@ -1024,53 +925,82 @@ def _build_server(graph_path: str): from graphify import paths as _paths - # Per-graph context cache: resolved graph.json path -> {key, G, communities}. - # The server's default graph is just the first entry; a tool call carrying a - # project_path adds its own. Routing every graph through one cache means the - # eager trigram index and the mtime+size hot-reload behave identically for - # the default graph and for any project graph. + # Per-store context: each reader retains one immutable native snapshot and + # its long-lived public predicate client until the generation changes. _default_graph_path = graph_path _ctx_lock = threading.Lock() + _tool_lock = threading.Lock() _ctx_cache: dict[str, dict] = {} def _load_ctx(path: str): - """Return (G, communities) for a graph.json path, reusing a cached - context until the file's (mtime, size) changes and then transparently - rebuilding it. Unlike ``_load_graph`` it never exits the process on a - missing/corrupt file — it raises, so a bad project_path surfaces as a - tool error instead of killing a server that is happily serving other - projects.""" - try: - s = Path(path).stat() - key = (s.st_mtime_ns, s.st_size) - except FileNotFoundError: - raise FileNotFoundError(f"graph.json not found: {path}") - ent = _ctx_cache.get(path) - if ent is not None and ent["key"] == key: - return ent["G"], ent["communities"] + """Return the current native graph and communities for a Helix store.""" with _ctx_lock: ent = _ctx_cache.get(path) - if ent is not None and ent["key"] == key: - return ent["G"], ent["communities"] # another thread built it - try: - new_G = _load_graph(path) - except SystemExit as e: # _load_graph exits on missing/corrupt file - raise RuntimeError(f"could not load graph.json at {path}") from e - # Warm the trigram index before exposing the graph so the first query - # against it is fast (same rationale as the original startup warm-up). - _get_trigram_index(new_G) - comm = _communities_from_graph(new_G) - _ctx_cache[path] = {"key": key, "G": new_G, "communities": comm} - return new_G, comm + if ent is None: + ent = {"reader": HelixGraphReader(validate_store_path(path))} + _ctx_cache[path] = ent + loaded = ent["reader"].get() + if ent.get("generation") == loaded.generation: + return ( + ent["G"], + ent["native_query"], + ent["communities"], + ent["community_labels"], + ent["learning_overlay"], + ent["confidence_counts"], + ) + new_G = loaded.graph + learning = loaded.state.get("learning", {}) + learning_overlay = ( + dict(learning.get("nodes", {})) if isinstance(learning, dict) else {} + ) + community_labels = { + int(record["id"]): str(record.get("name") or f"Community {record['id']}") + for record in loaded.state.get("communities", []) + if isinstance(record, dict) and isinstance(record.get("id"), int) + } + comm = _communities_from_graph(loaded) + analysis = loaded.state.get("analysis", {}) + raw_counts = analysis.get("confidence_counts", {}) if isinstance(analysis, dict) else {} + confidence_counts = { + name: int(raw_counts.get(name, 0)) + for name in ("EXTRACTED", "INFERRED", "AMBIGUOUS") + } if isinstance(raw_counts, dict) else {} + if not confidence_counts or sum(confidence_counts.values()) != new_G.edge_count: + confidence_counts = { + "EXTRACTED": new_G.edge_count, + "INFERRED": 0, + "AMBIGUOUS": 0, + } + ent.update({ + "generation": loaded.generation, + "G": new_G, + "native_query": loaded.query, + "communities": comm, + "community_labels": community_labels, + "learning_overlay": learning_overlay, + "confidence_counts": confidence_counts, + }) + return ( + new_G, + loaded.query, + comm, + community_labels, + learning_overlay, + confidence_counts, + ) def _resolve_graph_path(project_path) -> str: - """Map an optional project_path to a concrete graph.json path. ``None`` + """Map an optional project_path to a concrete Helix store. ``None`` keeps the server's default graph (backward-compatible); a project_path - resolves to ``//graph.json``, honouring the + resolves to ``//graph.helix``, honouring the GRAPHIFY_OUT override so worktree/shared-output setups keep working.""" if not project_path: return _default_graph_path - return str(Path(project_path) / _paths.GRAPHIFY_OUT / "graph.json") + candidate = Path(project_path) + if candidate.name == "graph.helix": + return str(candidate) + return str(candidate / _paths.GRAPHIFY_OUT / "graph.helix") # Active per-request context, rebound by _select_graph() and read by the tool # handlers below. No lock needed on the hot path: _select_graph and the @@ -1079,17 +1009,35 @@ def _resolve_graph_path(project_path) -> str: # swap. active_graph_path = _default_graph_path try: - G, communities = _load_ctx(_default_graph_path) + ( + G, + native_query, + communities, + community_labels, + learning_overlay, + confidence_counts, + ) = _load_ctx(_default_graph_path) except (FileNotFoundError, RuntimeError): # No default graph at startup → run as a pure multi-project server. Tools # then require project_path; a call without one gets a clear error rather # than the process refusing to start (which is what _load_graph would do). - G, communities = None, {} + G, native_query, communities, community_labels, learning_overlay, confidence_counts = ( + None, None, {}, {}, {}, {} + ) def _select_graph(project_path) -> None: - nonlocal G, communities, active_graph_path + nonlocal G, native_query, communities, community_labels, learning_overlay + nonlocal confidence_counts + nonlocal active_graph_path path = _resolve_graph_path(project_path) - G, communities = _load_ctx(path) + ( + G, + native_query, + communities, + community_labels, + learning_overlay, + confidence_counts, + ) = _load_ctx(path) active_graph_path = path server = Server("graphify") @@ -1226,7 +1174,7 @@ async def list_tools() -> list[types.Tool]: "type": "string", "description": ( "Absolute path to a project directory containing " - "graphify-out/graph.json. Optional — defaults to the graph " + "graphify-out/graph.helix. Optional — defaults to the graph " "this server was started with." ), } @@ -1244,10 +1192,12 @@ def _tool_query_graph(arguments: dict) -> str: result = _query_graph_text( G, question, + native_query=native_query, mode=mode, depth=depth, token_budget=budget, context_filters=context_filter, + learning_overlay=learning_overlay, ) querylog.log_query( kind="mcp_query", @@ -1263,36 +1213,36 @@ def _tool_query_graph(arguments: dict) -> str: def _tool_get_node(arguments: dict) -> str: label = arguments["label"].lower() - matches = [(nid, d) for nid, d in G.nodes(data=True) - if label in (d.get("label") or "").lower() or label == nid.lower()] + matches = _find_node(G, label, native_query=native_query) if not matches: return f"No node matching '{label}' found." - nid, d = matches[0] + nid = matches[0] + d = node_attributes(G, nid) # Sanitise every LLM-derived field before concatenation (F-010). return "\n".join([ f"Node: {sanitize_label(d.get('label', nid))}", - f" ID: {sanitize_label(nid)}", + f" ID: {sanitize_label(str(nid))}", f" Source: {sanitize_label(str(d.get('source_file', '')))} {sanitize_label(str(d.get('source_location', '')))}", f" Type: {sanitize_label(str(d.get('file_type', '')))}", f" Community: {sanitize_label(str(d.get('community_name') or d.get('community', '')))}", - f" Degree: {G.degree(nid)}", + f" Degree: {G.degree(nid).degree}", ]) def _tool_get_neighbors(arguments: dict) -> str: label = arguments["label"].lower() rel_filter = arguments.get("relation_filter", "").lower() - matches = _find_node(G, label) + matches = _find_node(G, label, native_query=native_query) if not matches: return f"No node matching '{label}' found." nid = matches[0] - lines = [f"Neighbors of {sanitize_label(G.nodes[nid].get('label', nid))}:"] + lines = [f"Neighbors of {sanitize_label(node_attributes(G, nid).get('label', nid))}:"] for nb in G.successors(nid): d = edge_data(G, nid, nb) rel = d.get("relation", "") - if rel_filter and rel_filter not in rel.lower(): + if rel_filter and rel_filter not in str(rel).lower(): continue lines.append( - f" --> {sanitize_label(G.nodes[nb].get('label', nb))} " + f" --> {sanitize_label(node_attributes(G, nb).get('label', nb))} " f"[{sanitize_label(str(rel))}] [{sanitize_label(str(d.get('confidence', '')))}]" ) for nb in G.predecessors(nid): @@ -1301,7 +1251,7 @@ def _tool_get_neighbors(arguments: dict) -> str: if rel_filter and rel_filter not in rel.lower(): continue lines.append( - f" <-- {sanitize_label(G.nodes[nb].get('label', nb))} " + f" <-- {sanitize_label(node_attributes(G, nb).get('label', nb))} " f"[{sanitize_label(str(rel))}] [{sanitize_label(str(d.get('confidence', '')))}]" ) return "\n".join(lines) @@ -1311,10 +1261,10 @@ def _tool_get_community(arguments: dict) -> str: nodes = communities.get(cid, []) if not nodes: return f"Community {cid} not found." - header = _community_header(cid, G.nodes[nodes[0]].get("community_name")) + header = _community_header(cid, node_attributes(G, nodes[0]).get("community_name")) lines = [f"{header} ({len(nodes)} nodes):"] for n in nodes: - d = G.nodes[n] + d = node_attributes(G, n) # Sanitise label and source_file (F-010). lines.append( f" {sanitize_label(d.get('label', n))} " @@ -1330,20 +1280,27 @@ def _tool_god_nodes(arguments: dict) -> str: return "\n".join(lines) def _tool_graph_stats(_: dict) -> str: - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 + total = G.edge_count or 1 return ( - f"Nodes: {G.number_of_nodes()}\n" - f"Edges: {G.number_of_edges()}\n" + f"Nodes: {G.node_count}\n" + f"Edges: {G.edge_count}\n" f"Communities: {len(communities)}\n" - f"EXTRACTED: {round(confs.count('EXTRACTED')/total*100)}%\n" - f"INFERRED: {round(confs.count('INFERRED')/total*100)}%\n" - f"AMBIGUOUS: {round(confs.count('AMBIGUOUS')/total*100)}%\n" + f"EXTRACTED: {round(confidence_counts['EXTRACTED']/total*100)}%\n" + f"INFERRED: {round(confidence_counts['INFERRED']/total*100)}%\n" + f"AMBIGUOUS: {round(confidence_counts['AMBIGUOUS']/total*100)}%\n" ) def _tool_shortest_path(arguments: dict) -> str: - src_scored = _score_nodes(G, [t.lower() for t in arguments["source"].split()]) - tgt_scored = _score_nodes(G, [t.lower() for t in arguments["target"].split()]) + src_scored = _score_nodes( + G, + [t.lower() for t in arguments["source"].split()], + native_query=native_query, + ) + tgt_scored = _score_nodes( + G, + [t.lower() for t in arguments["target"].split()], + native_query=native_query, + ) if not src_scored: return f"No node matching source '{arguments['source']}' found." if not tgt_scored: @@ -1375,16 +1332,19 @@ def _tool_shortest_path(arguments: dict) -> str: max_hops = int(arguments.get("max_hops", 8)) try: # Use undirected view for path-finding (works regardless of query src/tgt order) - path_nodes = nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) - except (nx.NetworkXNoPath, nx.NodeNotFound): - return f"No path found between '{G.nodes[src_nid].get('label', src_nid)}' and '{G.nodes[tgt_nid].get('label', tgt_nid)}'." + path_result = G.shortest_path(src_nid, tgt_nid, direction="both") + path_nodes = list(path_result.node_ids) + if not path_nodes: + raise ValueError("no path") + except (KeyError, ValueError, RuntimeError): + return f"No path found between '{node_attributes(G, src_nid).get('label', src_nid)}' and '{node_attributes(G, tgt_nid).get('label', tgt_nid)}'." hops = len(path_nodes) - 1 if hops > max_hops: return f"Path exceeds max_hops={max_hops} ({hops} hops found)." segments = [] for i in range(len(path_nodes) - 1): u, v = path_nodes[i], path_nodes[i + 1] - if G.has_edge(u, v): + if G.has_edge_between(u, v): edata = edge_data(G, u, v) forward = True else: @@ -1394,11 +1354,11 @@ def _tool_shortest_path(arguments: dict) -> str: conf = edata.get("confidence", "") conf_str = f" [{conf}]" if conf else "" if i == 0: - segments.append(G.nodes[u].get("label", u)) + segments.append(node_attributes(G, u).get("label", u)) if forward: - segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") + segments.append(f"--{rel}{conf_str}--> {node_attributes(G, v).get('label', v)}") else: - segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") + segments.append(f"<--{rel}{conf_str}-- {node_attributes(G, v).get('label', v)}") prefix = ("\n".join(warnings) + "\n") if warnings else "" return prefix + f"Shortest path ({hops} hops):\n " + " ".join(segments) @@ -1430,7 +1390,9 @@ def _tool_get_pr_impact(arguments: dict) -> str: files = fetch_pr_files(number, repo) if not files: return f"PR #{number}: no changed files found (may require gh auth)." - comms, nodes = compute_pr_impact(files, G) + comms, nodes = compute_pr_impact( + files, G, communities, native_query=native_query + ) ci = _parse_ci(pr_data.get("statusCheckRollup") or []) lines = [ f"PR #{number}: {pr_data['title']}", @@ -1472,7 +1434,9 @@ def _tool_triage_prs(arguments: dict) -> str: files = [] if files: pr.files_changed = files - pr.communities_touched, pr.nodes_affected = compute_pr_impact(files, G) + pr.communities_touched, pr.nodes_affected = compute_pr_impact( + files, G, communities, native_query=native_query + ) header = ( f"Actionable PRs targeting {base}: {len(actionable)}\n" "Rank these by review priority. Higher blast_radius = more graph communities affected = higher merge risk.\n" @@ -1501,13 +1465,9 @@ def _tool_triage_prs(arguments: dict) -> str: } def _load_community_labels() -> dict[int, str]: - labels_path = Path(active_graph_path).parent / ".graphify_labels.json" - if labels_path.exists(): - try: - return {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} - except Exception: - pass - return {cid: f"Community {cid}" for cid in communities} + return dict(community_labels) or { + cid: f"Community {cid}" for cid in communities + } @server.list_resources() async def list_resources() -> list[types.Resource]: @@ -1522,66 +1482,92 @@ async def list_resources() -> list[types.Resource]: @server.read_resource() async def read_resource(uri: AnyUrl) -> str: - _select_graph(None) # resources read the server's default graph - uri_str = str(uri) - if uri_str == "graphify://report": - report_path = Path(active_graph_path).parent / "GRAPH_REPORT.md" - if report_path.exists(): - return report_path.read_text(encoding="utf-8") - return "GRAPH_REPORT.md not found. Run graphify extract first." - if uri_str == "graphify://stats": - return _tool_graph_stats({}) - if uri_str == "graphify://god-nodes": - return _tool_god_nodes({"top_n": 10}) - if uri_str == "graphify://surprises": - try: - from graphify.analyze import surprising_connections - surprises = surprising_connections(G, communities, top_n=10) - if not surprises: - return "No surprising connections found." - lines = ["Surprising cross-community connections:"] - for s in surprises: - lines.append(f" {s.get('source', '')} <-> {s.get('target', '')} [{s.get('relation', '')}]") - return "\n".join(lines) - except Exception as exc: - return f"Could not compute surprising connections: {exc}" - if uri_str == "graphify://audit": - confs = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)] - total = len(confs) or 1 - return ( - f"Total edges: {total}\n" - f"EXTRACTED: {confs.count('EXTRACTED')} ({round(confs.count('EXTRACTED')/total*100)}%)\n" - f"INFERRED: {confs.count('INFERRED')} ({round(confs.count('INFERRED')/total*100)}%)\n" - f"AMBIGUOUS: {confs.count('AMBIGUOUS')} ({round(confs.count('AMBIGUOUS')/total*100)}%)\n" - ) - if uri_str == "graphify://questions": - try: - from graphify.analyze import suggest_questions - community_labels = _load_community_labels() - questions = suggest_questions(G, communities, community_labels, top_n=10) - if not questions: - return "No suggested questions available." - lines = ["Suggested questions:"] - for q in questions: - if isinstance(q, dict): - lines.append(f" - {q.get('question', '')}") - else: - lines.append(f" - {q}") - return "\n".join(lines) - except Exception as exc: - return f"Could not generate questions: {exc}" - raise ValueError(f"Unknown resource: {uri_str}") + import asyncio + + def read() -> str: + with _tool_lock: + _select_graph(None) # resources read the server's default graph + uri_str = str(uri) + if uri_str == "graphify://report": + report_path = Path(active_graph_path).parent / "GRAPH_REPORT.md" + if report_path.exists(): + return report_path.read_text(encoding="utf-8") + return "GRAPH_REPORT.md not found. Run graphify extract first." + if uri_str == "graphify://stats": + return _tool_graph_stats({}) + if uri_str == "graphify://god-nodes": + return _tool_god_nodes({"top_n": 10}) + if uri_str == "graphify://surprises": + try: + from graphify.analyze import surprising_connections + + surprises = surprising_connections(G, communities, top_n=10) + if not surprises: + return "No surprising connections found." + lines = ["Surprising cross-community connections:"] + for item in surprises: + lines.append( + f" {item.get('source', '')} <-> " + f"{item.get('target', '')} [{item.get('relation', '')}]" + ) + return "\n".join(lines) + except Exception as exc: + return f"Could not compute surprising connections: {exc}" + if uri_str == "graphify://audit": + total = G.edge_count or 1 + return ( + f"Total edges: {total}\n" + f"EXTRACTED: {confidence_counts['EXTRACTED']} " + f"({round(confidence_counts['EXTRACTED'] / total * 100)}%)\n" + f"INFERRED: {confidence_counts['INFERRED']} " + f"({round(confidence_counts['INFERRED'] / total * 100)}%)\n" + f"AMBIGUOUS: {confidence_counts['AMBIGUOUS']} " + f"({round(confidence_counts['AMBIGUOUS'] / total * 100)}%)\n" + ) + if uri_str == "graphify://questions": + try: + from graphify.analyze import suggest_questions + + labels = _load_community_labels() + questions = suggest_questions( + G, communities, labels, top_n=10 + ) + if not questions: + return "No suggested questions available." + lines = ["Suggested questions:"] + for question in questions: + if isinstance(question, dict): + lines.append(f" - {question.get('question', '')}") + else: + lines.append(f" - {question}") + return "\n".join(lines) + except Exception as exc: + return f"Could not generate questions: {exc}" + raise ValueError(f"Unknown resource: {uri_str}") + + return await asyncio.to_thread(read) @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[types.TextContent]: + import asyncio + arguments = dict(arguments or {}) project_path = arguments.pop("project_path", None) handler = _handlers.get(name) if not handler: return [types.TextContent(type="text", text=f"Unknown tool: {name}")] + + def _execute() -> str: + # Helix's embedded client is synchronous. Run selection and the tool + # as one serialized worker-thread operation so HTTP event loops are + # never blocked and closure-scoped active context cannot cross calls. + with _tool_lock: + _select_graph(project_path) + return handler(arguments) + try: - _select_graph(project_path) # bind G/communities to the target graph - return [types.TextContent(type="text", text=handler(arguments))] + result = await asyncio.to_thread(_execute) + return [types.TextContent(type="text", text=result)] except Exception as exc: return [types.TextContent(type="text", text=f"Error executing {name}: {exc}")] @@ -1590,7 +1576,7 @@ async def call_tool(name: str, arguments: dict) -> list[types.TextContent]: def serve(graph_path: str | None = None) -> None: """Start the MCP server over stdio (the default, per-developer transport).""" - graph_path = graph_path or _default_graph_json() + graph_path = graph_path or str(DEFAULT_PROJECT_STORE) try: from mcp.server.stdio import stdio_server except ImportError as e: @@ -1710,7 +1696,7 @@ def _build_http_app( # are intentionally exposing the server, so accept any Host header; for a # loopback/specific bind, restrict Host to that address (with and without # the port) plus the localhost aliases. - if host in ("0.0.0.0", "::", ""): + if host in ("0.0.0.0", "::", ""): # nosec B104 - explicit operator-selected bind security = TransportSecuritySettings(enable_dns_rebinding_protection=False) else: allowed = {host, "localhost", "127.0.0.1"} @@ -1768,7 +1754,7 @@ def serve_http( deliberate follow-up. Binding ``0.0.0.0`` exposes the server beyond localhost — set an api_key when you do. """ - graph_path = graph_path or _default_graph_json() + graph_path = graph_path or str(DEFAULT_PROJECT_STORE) try: import uvicorn except ImportError as e: @@ -1795,9 +1781,9 @@ def serve_http( f"graphify MCP server (streamable-http) on http://{host}:{port}{path} - {auth_note}", file=sys.stderr, ) - if host in ("0.0.0.0", "::", "") and not api_key: + if host in ("0.0.0.0", "::", "") and not api_key: # nosec B104 - emits warning print( - f"WARNING: binding {host or '0.0.0.0'} with no api-key exposes the graph " + f"WARNING: binding {host or '0.0.0.0'} with no api-key exposes the graph " # nosec B104 "unauthenticated on the network. Set --api-key (or GRAPHIFY_API_KEY).", file=sys.stderr, ) @@ -1816,14 +1802,14 @@ def _main(argv: list[str] | None = None) -> None: "graph_path", nargs="?", default=None, - help="Path to graph.json (default: graphify-out/graph.json)", + help="Path to graph.helix store (default: graphify-out/graph.helix)", ) parser.add_argument( "--graph", dest="graph_flag", default=None, metavar="PATH", - help="Path to graph.json — alias for the positional argument", + help="Path to graph.helix — alias for the positional argument", ) parser.add_argument( "--transport", @@ -1856,7 +1842,7 @@ def _main(argv: list[str] | None = None) -> None: help="Reap stateful sessions idle this many seconds (default: 3600; 0 disables)", ) args = parser.parse_args(argv) - graph_path = args.graph_flag or args.graph_path or _default_graph_json() + graph_path = args.graph_flag or args.graph_path or str(DEFAULT_PROJECT_STORE) if args.transport == "http": serve_http( diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 19d5bea4e..076572c86 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,408 +135,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 8db8f73f2..546030d11 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -5,1261 +5,128 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Graphify uses `graphify-out/graph.helix` as its only runtime store. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```bash +graphify extract [path] +graphify update [path] +graphify query "question" +graphify path "A" "B" +graphify explain "node" ``` ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Use native topology, analysis, confidence, and source locations to understand a corpus without re-reading it. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. +If the native store exists, query it. If only an obsolete-format file exists, ignore it and rebuild from source. Native Windows x86_64 is supported through the matching public package wheel. ### Step 1 - Ensure graphify is installed -```bash -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) -PYTHON="" -GRAPHIFY_BIN=$(which graphify 2>/dev/null) -# 1. uv tool installs — most reliable on modern Mac/Linux -if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi -fi -# 2. Read shebang from graphify binary (pipx and direct pip installs) -if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$_SHEBANG" in - *[!a-zA-Z0-9/_.@-]*) ;; - *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; - esac -fi -# 3. Fall back to python3 -if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -if ! "$PYTHON" -c "import graphify" 2>/dev/null; then - if command -v uv >/dev/null 2>&1; then - uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi - else - "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ - || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 - fi -fi -# Write interpreter path for all subsequent steps (persists across invocations) -mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -``` - -If the import succeeds, print nothing and move straight to Step 2. - -**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +Use `uv tool install graphifyy` or `python3 -m pip install graphifyy`. Homebrew is not required. ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: - -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) -``` - -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +The production CLI performs corpus detection and sensitive-file exclusion. ### Step 2.5 - Transcribe video / audio files (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Transcribe media to source text only when requested. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) is done by your own model. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you cannot dispatch subagents, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If this host cannot dispatch subagents, run the deterministic CLI path and skip optional semantic enrichment. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` +Code extraction is deterministic and keyless. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -> **Aider platform:** Multi-agent support is still early on Aider. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. - -Print: `"Semantic extraction: N files (sequential — Aider)"` - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('.graphify_detect.json').read_text()) -# Only content files go to semantic extraction. Code is already covered -# structurally by the AST pass; flattening every category here makes the -# extraction step re-read every source file (#1392). -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) - -# Always (re)write the cache file: write hits, else DELETE any leftover from a -# prior run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -else: - Path('.graphify_cached.json').unlink(missing_ok=True) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. - -**Step B2 - Sequential extraction (Aider)** - -Process each file one at a time. For each file: - -1. Read the file contents -2. Extract nodes, edges, and hyperedges applying the same rules: - - EXTRACTED: relationship explicit in source (import, call, citation) - - INFERRED: reasonable inference (shared structure, implied dependency) - - AMBIGUOUS: uncertain — flag it, do not omit - - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. - - Image files: use vision — understand what the image IS, not just OCR - - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. - - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. - - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 -3. Accumulate results across all files - -Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -After processing all files, write the accumulated result to `.graphify_semantic_new.json`. - -**Step B3 - Cache and merge** - -For the accumulated result: - -If more than half the chunks failed, stop and tell the user. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text()) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Use bounded semantic chunks only for non-code content. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +Transient DTOs are validated by the parent and committed with native state. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Persist the graph first and only write the report/analysis if it actually -# persisted - to_json refuses to shrink an existing graph.json (#479), and a -# report describing a graph we did not write would be a lie (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +Run `graphify extract INPUT_PATH`. Atomic activation deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Run `graphify label . --missing-only`; labels remain inside Helix state. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` +Run native `graphify export` commands. ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. +Run `graphify export neo4j graphify-out/graph.helix`. ### Step 7b - SVG export (only if --svg flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` +Run `graphify export svg graphify-out/graph.helix`. ### Step 7c - GraphML export (only if --graphml flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` +Run `graphify export graphml graphify-out/graph.helix`. ### Step 7d - MCP server (only if --mcp flag) -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` +Run `python3 -m graphify.serve graphify-out/graph.helix`. ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- +Run `graphify benchmark --graph graphify-out/graph.helix`. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Build metadata, hashes, caches, and learning state are durable native state. ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- +Run `graphify update INPUT_PATH`. ## For --cluster-only -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- +Run `graphify cluster-only INPUT_PATH`. ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - ---- +Run `graphify query "QUESTION" [--dfs] [--budget N]`. ## For /graphify path -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Run `graphify path "A" "B" --graph graphify-out/graph.helix`. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- +Run `graphify explain "NODE" --graph graphify-out/graph.helix`. ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- +Run `graphify add URL`, then `graphify update .`. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- +Run `graphify watch INPUT_PATH`. ## For git commit hook -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- +Run `graphify hook install`. ## For native CLAUDE.md integration -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` - ---- +Run `graphify claude install`. ## Honesty Rules -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never invent an edge. +- Never reconstruct or migrate obsolete runtime data. +- Preserve confidence tags and source citations. diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 19d5bea4e..076572c86 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,408 +135,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index bf9dd3431..b0fcd6f0d 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 04eb5705b..7d0f2d6b0 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message (Codex)** @@ -270,408 +135,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the compact subagent prompt (rules, node-ID format, confidence rubric, hyperedge and vision rules, JSON schema). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and have it return the JSON inline. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index bf9dd3431..b0fcd6f0d 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index c444a852d..1b024b042 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -3,1390 +3,142 @@ name: graphify description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." argument-hint: "[path|query|subcommand]" model: sonnet -allowed-tools: - - read - - grep - - glob - - exec -triggers: - - user - - model +allowed-tools: [read, grep, glob, exec] +triggers: [user, model] --- # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Graphify uses `graphify-out/graph.helix` as its only runtime store. ## Usage -``` -/graphify # full pipeline on current directory -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```bash +graphify extract [path] +graphify update [path] +graphify query "question" +graphify path "A" "B" +graphify explain "node" ``` ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes -> one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Use native topology, analysis, confidence, and source locations to understand a corpus without re-reading it. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. +If the native store exists, query it. If only an obsolete-format file exists, ignore it and rebuild from source. Native Windows x86_64 is supported through the matching public package wheel. ### Step 1 - Ensure graphify is installed -```bash -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) -PYTHON="" -GRAPHIFY_BIN=$(which graphify 2>/dev/null) -# 1. uv tool installs — most reliable on modern Mac/Linux -if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi -fi -# 2. Read shebang from graphify binary (pipx and direct pip installs) -if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$_SHEBANG" in - *[!a-zA-Z0-9/_.@-]*) ;; - *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; - esac -fi -# 3. Fall back to python3 -if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -if ! "$PYTHON" -c "import graphify" 2>/dev/null; then - if command -v uv >/dev/null 2>&1; then - uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi - else - "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ - || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 - fi -fi -# Write interpreter path for all subsequent steps (persists across invocations) -mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -# Force UTF-8 I/O on Windows (prevents garbled CJK/non-ASCII output) -export PYTHONUTF8=1 -``` - -If the import succeeds, print nothing and move straight to Step 2. - -**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +Use `uv tool install graphifyy` or `python3 -m pip install graphifyy`. Homebrew is not required. ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > graphify-out/.graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: - -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) -``` - -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +The production CLI performs corpus detection and sensitive-file exclusion. ### Step 2.5 - Transcribe video / audio files (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Transcribe media to source text only when requested. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) is done by your own model. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you cannot dispatch subagents, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If this host cannot dispatch subagents, run the deterministic CLI path and skip optional semantic enrichment. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` +Code extraction is deterministic and keyless. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**MANDATORY: You MUST use the subagent system here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use parallel subagents you are doing this wrong.** - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -# Only content files go to semantic extraction. Code is already covered -# structurally by the AST pass; flattening every category here makes the -# extraction step re-read every source file (#1392). -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) - -# Always (re)write the cache file: write hits, else DELETE any leftover from a -# prior run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. - -**Step B2 - Dispatch subagents** - -Dispatch ALL subagents in the same response so they run in parallel. - -Concrete example for 3 chunks: -``` -[Subagent 1: files 1-15] -[Subagent 2: files 16-30] -[Subagent 3: files 31-45] -``` -All three in one message. Not three separate messages. - -For each chunk, dispatch a subagent with this exact prompt (fill in FILE_LIST): - -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON with no commentary: {"nodes": [...], "edges": [...], "hyperedges": [...], "input_tokens": 0, "output_tokens": 0} - -Extraction rules: -- EXTRACTED: relationship explicit in source (import, call, citation) -- INFERRED: reasonable inference (shared structure, implied dependency) -- AMBIGUOUS: uncertain — flag it, do not omit -- Code files: extract semantic edges AST cannot find (design patterns, protocol conformance). Do not re-extract imports. -- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. -- Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. -- DEEP_MODE (if set): be aggressive with INFERRED edges -- Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. -- Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. -- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. -- confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: pick exactly ONE value from this set — never 0.5: - 0.95 direct structural evidence (shared data structure, named cross-file reference). - 0.85 strong inference (clear functional alignment, no direct symbol link). - 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). - 0.65 weak inference (thematically related, no shape evidence). - 0.55 speculative but plausible (surface-level co-occurrence only). - Models follow discrete rubrics better than continuous ranges; the bimodal - distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the - range guidance is being collapsed to a binary. If no value above fits, mark - the edge AMBIGUOUS rather than picking 0.4 or below. -- AMBIGUOUS edges: 0.1-0.3 - -Schema: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -Files: -FILE_LIST -``` - -**Step B3 - Cache and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_N.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -After each subagent call completes, write its result to `graphify-out/.graphify_chunk_N.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then merge: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path -from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d, errors = load_validated_semantic_fragment(Path(c)) - if errors: - print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) - continue - d = sanitize_semantic_fragment(d) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Use bounded semantic chunks only for non-code content. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +Transient DTOs are validated by the parent and committed with native state. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Persist the graph first and only write the report/analysis if it actually -# persisted - to_json refuses to shrink an existing graph.json (#479), and a -# report describing a graph we did not write would be a lie (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +Run `graphify extract INPUT_PATH`. Atomic activation deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Run `graphify label . --missing-only`; labels remain inside Helix state. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -NODE_LIMIT = 5000 -if G.number_of_nodes() > NODE_LIMIT: - from collections import Counter - print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - import networkx as nx_meta - meta = nx_meta.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) - edge_counts = Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') - if meta.number_of_nodes() > 1: - meta_communities = {cid: [str(cid)] for cid in communities} - member_counts = {cid: len(members) for cid, members in communities.items()} - to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) - print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') - print('Tip: run with --obsidian for full node-level detail, or --wiki for an agent-crawlable wiki.') - else: - print('Single community — aggregated view not useful. Skipping graph.html.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` +Run native `graphify export` commands. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -The wiki is an agent-crawlable export — `index.md` plus one article per community plus god-node articles. It is the recommended fallback for graphs too large to render as HTML, and it's the most useful output for an autonomous agent navigating the graph between sessions. - -Run this before Step 9 (cleanup) so `graphify-out/.graphify_labels.json` is still available. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.wiki import to_wiki -from graphify.analyze import god_nodes -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} -gods = god_nodes(G) - -n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) -print(f'Wiki: {n} articles written to graphify-out/wiki/') -print(' graphify-out/wiki/index.md -> agent entry point') -" -``` +Run `graphify export wiki graphify-out/graph.helix`. ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. +Run `graphify export neo4j graphify-out/graph.helix`. ### Step 7b - SVG export (only if --svg flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` +Run `graphify export svg graphify-out/graph.helix`. ### Step 7c - GraphML export (only if --graphml flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` +Run `graphify export graphml graphify-out/graph.helix`. ### Step 7d - MCP server (only if --mcp flag) -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` +Run `python3 -m graphify.serve graphify-out/graph.helix`. ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- +Run `graphify benchmark --graph graphify-out/graph.helix`. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) - wiki/ - agent-crawlable wiki, start at wiki/index.md (only if --wiki was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Build metadata, hashes, caches, and learning state are durable native state. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" -fi -``` - ---- +Prefer `graphify`; otherwise use `python3 -m graphify`. ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4-8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A-3C pipeline as normal. - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4-8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` - ---- +Run `graphify update INPUT_PATH`. ## For --cluster-only -Skip Steps 1-3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5-9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- +Run `graphify cluster-only INPUT_PATH`. ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - ---- +Run `graphify query "QUESTION" [--dfs] [--budget N]`. ## For /graphify path -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Run `graphify path "A" "B" --graph graphify-out/graph.helix`. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept. Then write a 3-5 sentence explanation using source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- +Run `graphify explain "NODE" --graph graphify-out/graph.helix`. ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv -> abstract + metadata saved as `.md` -- PDF -> downloaded as `.pdf` -- Images (.png/.jpg/.webp) -> downloaded, vision extraction runs on next build -- Any webpage -> converted to markdown via html2text - ---- +Run `graphify add URL`, then `graphify update .`. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - ---- +Run `graphify watch INPUT_PATH`. ## For git commit hook -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed, re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - ---- +Run `graphify hook install`. ## For always-on context in Devin sessions -Run once per project to make graphify always-on in Devin sessions: - -```bash -graphify devin install --project -``` - -This writes a `## graphify` section to `.windsurf/rules/graphify.md` that instructs Devin to check the graph before answering codebase questions and rebuild it after code changes. - -```bash -graphify devin uninstall --project # remove -``` - ---- +Tell Devin to query `graphify-out/graph.helix` before broad source search. ## Honesty Rules -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. +- Never invent an edge. +- Never reconstruct or migrate obsolete runtime data. +- Preserve confidence tags and source citations. diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index ffa94995d..bfa1c994e 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,408 +135,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 96dfc2e85..34063ca95 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,401 +138,89 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- @@ -683,7 +236,6 @@ When the user asks to install the post-commit auto-rebuild hook or wire graphify ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index bf9dd3431..b0fcd6f0d 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 6613434e8..e7fc32f95 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message (OpenCode)** @@ -265,408 +130,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index bf9dd3431..b0fcd6f0d 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 407f091b6..097fbe45d 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -271,408 +136,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index f7e7f8442..ed606b3ee 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch subagents and paste their responses** @@ -269,408 +134,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and write its response to that chunk's file. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 9c7ab7420..47adbe667 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -126,148 +104,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent block, run Python through the saved interpreter — `& (Get-Content graphify-out\.graphify_python)` in place of a bare `python3` — so every step uses the interpreter that actually has graphify.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -295,421 +160,107 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Troubleshooting -### PowerShell 5.1: Vertical scrolling stops working +### Windows support -If vertical scrolling breaks in PowerShell after running graphify, this is caused by ANSI escape sequences from the `graspologic` library. Graphify v0.3.10+ suppresses this output, but if you still see the issue: +Use CPython 3.10 or 3.12 on native Windows x86_64 and install Graphify normally with pip or uv. The exact public `helix-db-embedded` version must provide a `win_amd64` wheel; do not substitute WSL, a source build, a downloaded DLL, or a compatibility graph library. + +### PowerShell 5.1: Vertical scrolling stops working -1. **Upgrade graphify**: `pip install --upgrade graphifyy` -2. **Use Windows Terminal** instead of the legacy PowerShell console — Windows Terminal handles ANSI codes correctly -3. **Reset your terminal**: close and reopen PowerShell -4. **Skip graspologic**: uninstall it (`pip uninstall graspologic`) and graphify will fall back to NetworkX's built-in Louvain algorithm, which produces no ANSI output +Use Windows Terminal or PowerShell 7 when possible. Graphify does not patch terminal modes or load a helper DLL. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skill.md b/graphify/skill.md index bf9dd3431..b0fcd6f0d 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/graphify/skills/agents/references/add-watch.md b/graphify/skills/agents/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/agents/references/add-watch.md +++ b/graphify/skills/agents/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/agents/references/exports.md b/graphify/skills/agents/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/agents/references/exports.md +++ b/graphify/skills/agents/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/agents/references/github-and-merge.md b/graphify/skills/agents/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/agents/references/github-and-merge.md +++ b/graphify/skills/agents/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/agents/references/hooks.md b/graphify/skills/agents/references/hooks.md index 3fb74d154..4f22e262c 100644 --- a/graphify/skills/agents/references/hooks.md +++ b/graphify/skills/agents/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/agents/references/query.md b/graphify/skills/agents/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/agents/references/query.md +++ b/graphify/skills/agents/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/agents/references/update.md b/graphify/skills/agents/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/agents/references/update.md +++ b/graphify/skills/agents/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/amp/references/add-watch.md b/graphify/skills/amp/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/amp/references/add-watch.md +++ b/graphify/skills/amp/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/amp/references/exports.md b/graphify/skills/amp/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/amp/references/exports.md +++ b/graphify/skills/amp/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/amp/references/github-and-merge.md b/graphify/skills/amp/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/amp/references/github-and-merge.md +++ b/graphify/skills/amp/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/amp/references/hooks.md b/graphify/skills/amp/references/hooks.md index af1ac7e72..9e318be21 100644 --- a/graphify/skills/amp/references/hooks.md +++ b/graphify/skills/amp/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/amp/references/query.md b/graphify/skills/amp/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/amp/references/query.md +++ b/graphify/skills/amp/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/amp/references/update.md b/graphify/skills/amp/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/amp/references/update.md +++ b/graphify/skills/amp/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/claude/references/add-watch.md b/graphify/skills/claude/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/claude/references/add-watch.md +++ b/graphify/skills/claude/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/claude/references/exports.md b/graphify/skills/claude/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/claude/references/exports.md +++ b/graphify/skills/claude/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/claude/references/github-and-merge.md b/graphify/skills/claude/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/claude/references/github-and-merge.md +++ b/graphify/skills/claude/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/claude/references/hooks.md b/graphify/skills/claude/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/claude/references/hooks.md +++ b/graphify/skills/claude/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/claude/references/query.md b/graphify/skills/claude/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/claude/references/query.md +++ b/graphify/skills/claude/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/claude/references/update.md b/graphify/skills/claude/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/claude/references/update.md +++ b/graphify/skills/claude/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/claw/references/add-watch.md b/graphify/skills/claw/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/claw/references/add-watch.md +++ b/graphify/skills/claw/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/claw/references/exports.md b/graphify/skills/claw/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/claw/references/exports.md +++ b/graphify/skills/claw/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/claw/references/github-and-merge.md b/graphify/skills/claw/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/claw/references/github-and-merge.md +++ b/graphify/skills/claw/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/claw/references/hooks.md b/graphify/skills/claw/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/claw/references/hooks.md +++ b/graphify/skills/claw/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/claw/references/query.md b/graphify/skills/claw/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/claw/references/query.md +++ b/graphify/skills/claw/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/claw/references/update.md b/graphify/skills/claw/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/claw/references/update.md +++ b/graphify/skills/claw/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/codex/references/add-watch.md b/graphify/skills/codex/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/codex/references/add-watch.md +++ b/graphify/skills/codex/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/codex/references/exports.md b/graphify/skills/codex/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/codex/references/exports.md +++ b/graphify/skills/codex/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/codex/references/github-and-merge.md b/graphify/skills/codex/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/codex/references/github-and-merge.md +++ b/graphify/skills/codex/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/codex/references/hooks.md b/graphify/skills/codex/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/codex/references/hooks.md +++ b/graphify/skills/codex/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/codex/references/query.md b/graphify/skills/codex/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/codex/references/query.md +++ b/graphify/skills/codex/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/codex/references/update.md b/graphify/skills/codex/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/codex/references/update.md +++ b/graphify/skills/codex/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/copilot/references/add-watch.md b/graphify/skills/copilot/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/copilot/references/add-watch.md +++ b/graphify/skills/copilot/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/copilot/references/exports.md b/graphify/skills/copilot/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/copilot/references/exports.md +++ b/graphify/skills/copilot/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/copilot/references/github-and-merge.md b/graphify/skills/copilot/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/copilot/references/github-and-merge.md +++ b/graphify/skills/copilot/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/copilot/references/hooks.md b/graphify/skills/copilot/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/copilot/references/hooks.md +++ b/graphify/skills/copilot/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/copilot/references/query.md b/graphify/skills/copilot/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/copilot/references/query.md +++ b/graphify/skills/copilot/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/copilot/references/update.md b/graphify/skills/copilot/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/copilot/references/update.md +++ b/graphify/skills/copilot/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/droid/references/add-watch.md b/graphify/skills/droid/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/droid/references/add-watch.md +++ b/graphify/skills/droid/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/droid/references/exports.md b/graphify/skills/droid/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/droid/references/exports.md +++ b/graphify/skills/droid/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/droid/references/github-and-merge.md b/graphify/skills/droid/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/droid/references/github-and-merge.md +++ b/graphify/skills/droid/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/droid/references/hooks.md b/graphify/skills/droid/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/droid/references/hooks.md +++ b/graphify/skills/droid/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/droid/references/query.md b/graphify/skills/droid/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/droid/references/query.md +++ b/graphify/skills/droid/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/droid/references/update.md b/graphify/skills/droid/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/droid/references/update.md +++ b/graphify/skills/droid/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/kilo/references/add-watch.md b/graphify/skills/kilo/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/kilo/references/add-watch.md +++ b/graphify/skills/kilo/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/kilo/references/exports.md b/graphify/skills/kilo/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/kilo/references/exports.md +++ b/graphify/skills/kilo/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/kilo/references/github-and-merge.md b/graphify/skills/kilo/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/kilo/references/github-and-merge.md +++ b/graphify/skills/kilo/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/kilo/references/hooks.md b/graphify/skills/kilo/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/kilo/references/hooks.md +++ b/graphify/skills/kilo/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/kilo/references/query.md b/graphify/skills/kilo/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/kilo/references/query.md +++ b/graphify/skills/kilo/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/kilo/references/update.md b/graphify/skills/kilo/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/kilo/references/update.md +++ b/graphify/skills/kilo/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/kiro/references/add-watch.md b/graphify/skills/kiro/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/kiro/references/add-watch.md +++ b/graphify/skills/kiro/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/kiro/references/exports.md b/graphify/skills/kiro/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/kiro/references/exports.md +++ b/graphify/skills/kiro/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/kiro/references/github-and-merge.md b/graphify/skills/kiro/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/kiro/references/github-and-merge.md +++ b/graphify/skills/kiro/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/kiro/references/hooks.md b/graphify/skills/kiro/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/kiro/references/hooks.md +++ b/graphify/skills/kiro/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/kiro/references/query.md b/graphify/skills/kiro/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/kiro/references/query.md +++ b/graphify/skills/kiro/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/kiro/references/update.md b/graphify/skills/kiro/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/kiro/references/update.md +++ b/graphify/skills/kiro/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/opencode/references/add-watch.md b/graphify/skills/opencode/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/opencode/references/add-watch.md +++ b/graphify/skills/opencode/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/opencode/references/exports.md b/graphify/skills/opencode/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/opencode/references/exports.md +++ b/graphify/skills/opencode/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/opencode/references/github-and-merge.md b/graphify/skills/opencode/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/opencode/references/github-and-merge.md +++ b/graphify/skills/opencode/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/opencode/references/hooks.md b/graphify/skills/opencode/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/opencode/references/hooks.md +++ b/graphify/skills/opencode/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/opencode/references/query.md b/graphify/skills/opencode/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/opencode/references/query.md +++ b/graphify/skills/opencode/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/opencode/references/update.md b/graphify/skills/opencode/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/opencode/references/update.md +++ b/graphify/skills/opencode/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/pi/references/add-watch.md b/graphify/skills/pi/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/pi/references/add-watch.md +++ b/graphify/skills/pi/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/pi/references/exports.md b/graphify/skills/pi/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/pi/references/exports.md +++ b/graphify/skills/pi/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/pi/references/github-and-merge.md b/graphify/skills/pi/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/pi/references/github-and-merge.md +++ b/graphify/skills/pi/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/pi/references/hooks.md b/graphify/skills/pi/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/pi/references/hooks.md +++ b/graphify/skills/pi/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/pi/references/query.md b/graphify/skills/pi/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/pi/references/query.md +++ b/graphify/skills/pi/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/pi/references/update.md b/graphify/skills/pi/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/pi/references/update.md +++ b/graphify/skills/pi/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/trae/references/add-watch.md b/graphify/skills/trae/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/trae/references/add-watch.md +++ b/graphify/skills/trae/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/trae/references/exports.md b/graphify/skills/trae/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/trae/references/exports.md +++ b/graphify/skills/trae/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/trae/references/github-and-merge.md b/graphify/skills/trae/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/trae/references/github-and-merge.md +++ b/graphify/skills/trae/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/trae/references/hooks.md b/graphify/skills/trae/references/hooks.md index 7c04d5b0b..04fbab01a 100644 --- a/graphify/skills/trae/references/hooks.md +++ b/graphify/skills/trae/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/trae/references/query.md b/graphify/skills/trae/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/trae/references/query.md +++ b/graphify/skills/trae/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/trae/references/update.md b/graphify/skills/trae/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/trae/references/update.md +++ b/graphify/skills/trae/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/vscode/references/add-watch.md b/graphify/skills/vscode/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/vscode/references/add-watch.md +++ b/graphify/skills/vscode/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/vscode/references/exports.md b/graphify/skills/vscode/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/vscode/references/exports.md +++ b/graphify/skills/vscode/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/vscode/references/github-and-merge.md b/graphify/skills/vscode/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/vscode/references/github-and-merge.md +++ b/graphify/skills/vscode/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/vscode/references/hooks.md b/graphify/skills/vscode/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/vscode/references/hooks.md +++ b/graphify/skills/vscode/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/vscode/references/query.md b/graphify/skills/vscode/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/vscode/references/query.md +++ b/graphify/skills/vscode/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/vscode/references/update.md b/graphify/skills/vscode/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/vscode/references/update.md +++ b/graphify/skills/vscode/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/skills/windows/references/add-watch.md b/graphify/skills/windows/references/add-watch.md index 77844343e..a6878f641 100644 --- a/graphify/skills/windows/references/add-watch.md +++ b/graphify/skills/windows/references/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/graphify/skills/windows/references/exports.md b/graphify/skills/windows/references/exports.md index 242ff868e..2d226ecb5 100644 --- a/graphify/skills/windows/references/exports.md +++ b/graphify/skills/windows/references/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/graphify/skills/windows/references/github-and-merge.md b/graphify/skills/windows/references/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/graphify/skills/windows/references/github-and-merge.md +++ b/graphify/skills/windows/references/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/graphify/skills/windows/references/hooks.md b/graphify/skills/windows/references/hooks.md index 438b8b16b..a908864c1 100644 --- a/graphify/skills/windows/references/hooks.md +++ b/graphify/skills/windows/references/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/graphify/skills/windows/references/query.md b/graphify/skills/windows/references/query.md index 56565eb78..082e28887 100644 --- a/graphify/skills/windows/references/query.md +++ b/graphify/skills/windows/references/query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/graphify/skills/windows/references/update.md b/graphify/skills/windows/references/update.md index fa2612180..740e6b139 100644 --- a/graphify/skills/windows/references/update.md +++ b/graphify/skills/windows/references/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/graphify/tree_html.py b/graphify/tree_html.py index 94b10df45..82ede1ae4 100644 --- a/graphify/tree_html.py +++ b/graphify/tree_html.py @@ -569,11 +569,22 @@ def write_tree_html( # kept for CLI compatibility with the older signature; ignored now top_k_edges: int = 0, ) -> Path: - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(graph_path) - graph = json.loads(graph_path.read_text(encoding="utf-8")) - tree = build_tree(graph, root=root, max_children=max_children, - project_label=project_label) + from graphify.helix.model import graphify_attributes + from graphify.helix.persistence import load_graph + from graphify.security import validate_store_path + + loaded = load_graph(validate_store_path(graph_path)) + tree = build_tree( + { + "nodes": [ + {"id": node.id, **graphify_attributes(node.attributes)} + for node in loaded.graph.nodes() + ] + }, + root=root, + max_children=max_children, + project_label=project_label, + ) title = f"{tree['name']} — graphify tree viewer" header = f"{tree['name']} — Knowledge Graph" html = emit_html(tree, title=title, header=header) diff --git a/graphify/validate.py b/graphify/validate.py index bab3ddc7c..b20d9ea52 100644 --- a/graphify/validate.py +++ b/graphify/validate.py @@ -51,7 +51,7 @@ def validate_extraction(data: dict) -> list[str]: f"'{node['file_type']}' - must be one of {sorted(VALID_FILE_TYPES)}" ) - # Edges - accept "links" (NetworkX <= 3.1) as fallback for "edges" + # Edges - accept the standard node-link spelling as a fallback for "edges". edge_list = data.get("edges") if "edges" in data else data.get("links") if edge_list is None: errors.append("Missing required key 'edges'") diff --git a/graphify/watch.py b/graphify/watch.py index 37edc9cb1..7abf49fe9 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -1,726 +1,249 @@ -# monitor a folder and auto-trigger --update when files change +"""Watch a project and atomically activate native Helix graph generations.""" + from __future__ import annotations + +import copy import contextlib +import hashlib import json import os -import posixpath -import re +import subprocess import sys import time +from contextlib import contextmanager from pathlib import Path -# Single source of truth in graphify.paths (#1423); re-exported as _GRAPHIFY_OUT. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from .helix.model import node_attributes +from .paths import GRAPHIFY_OUT as _GRAPHIFY_OUT + + _PENDING_FILENAME = ".pending_changes" _PENDING_DRAIN_MAX_PASSES = 20 -def _queue_pending(out_dir: Path, changed_paths: list[Path]) -> None: - """Append ``changed_paths`` to ``out_dir/.pending_changes`` (one per line). +from graphify.detect import ( # noqa: E402 + CODE_EXTENSIONS, + DOC_EXTENSIONS, + IMAGE_EXTENSIONS, + PAPER_EXTENSIONS, + _is_ignored, + _load_graphifyignore, +) - Used by a post-commit hook process that cannot acquire ``_rebuild_lock`` - so its change set is not silently dropped (#1059). The lock-holding - process drains this file before and after its rebuild and merges the - contents with its own change set. - Opened in append mode so concurrent writers do not clobber each other on - POSIX; each ``write()`` of a small payload is effectively atomic. A - trailing newline is always written so partial-line corruption stays - confined to the offending entry and is skipped on drain. - """ +_WATCHED_EXTENSIONS = CODE_EXTENSIONS | DOC_EXTENSIONS | PAPER_EXTENSIONS | IMAGE_EXTENSIONS +_CODE_EXTENSIONS = CODE_EXTENSIONS + + +def _topology_sources(build_data) -> list[str]: + """Return normalized source files that contributed native topology.""" + sources: set[str] = set() + + def add(attributes) -> None: + source = attributes.get("source_file") + if source: + sources.add(str(source).replace("\\", "/")) + + for node in build_data.nodes: + add(node.attributes) + for edge in build_data.edges: + add(edge.attributes) + hyperedges = build_data.attributes.get("hyperedges", []) + if isinstance(hyperedges, list): + for hyperedge in hyperedges: + if isinstance(hyperedge, dict): + add(hyperedge) + return sorted(sources) + + +def _queue_pending(out_dir: Path, changed_paths: list[Path]) -> None: + """Append an incremental change set for the active lock holder to drain.""" if not changed_paths: return out_dir.mkdir(parents=True, exist_ok=True) - pending = out_dir / _PENDING_FILENAME - payload = "".join(f"{os.fspath(p)}\n" for p in changed_paths) - with open(pending, "a", encoding="utf-8") as fh: - fh.write(payload) + payload = "".join(f"{os.fspath(path)}\n" for path in changed_paths) + with (out_dir / _PENDING_FILENAME).open("a", encoding="utf-8") as handle: + handle.write(payload) def _drain_pending(out_dir: Path) -> list[Path]: - """Read + unlink ``out_dir/.pending_changes`` and return deduplicated paths. - - Returns an empty list if the file does not exist. Empty/whitespace lines - are silently skipped so a partial concurrent write that left only a - fragment cannot poison the merge. - """ + """Atomically consume and de-duplicate queued incremental paths.""" pending = out_dir / _PENDING_FILENAME - if not pending.exists(): - return [] try: raw = pending.read_text(encoding="utf-8") - except OSError: + except (FileNotFoundError, OSError): return [] - # Unlink BEFORE returning so a crash between read and process retains the - # data in the next caller's view via the lines we are about to return — - # i.e. losing the file after reading is fine, losing it before would be a - # bug. Use missing_ok to tolerate a racing drain on platforms where - # rename/unlink may interleave. with contextlib.suppress(FileNotFoundError): pending.unlink() seen: set[str] = set() - out: list[Path] = [] + paths: list[Path] = [] for line in raw.splitlines(): - s = line.strip() - if not s or s in seen: - continue - seen.add(s) - out.append(Path(s)) - return out + value = line.strip() + if value and value not in seen: + seen.add(value) + paths.append(Path(value)) + return paths -# Build options that must survive into later rebuilds. The initial `extract` -# scan honours `--exclude`, but `update`/`watch`/hook rebuilds re-run detect() -# and would silently re-include excluded paths unless the patterns are persisted -# (#1886). We store them beside the graph so any rebuild driver can re-apply them. -_BUILD_CONFIG_FILENAME = ".graphify_build.json" +def _merge_changed_paths(*sources: list[Path] | None) -> list[Path]: + """Merge path lists in first-seen order.""" + seen: set[str] = set() + merged: list[Path] = [] + for source in sources: + for path in source or []: + key = os.fspath(path) + if key not in seen: + seen.add(key) + merged.append(path) + return merged def _write_build_config( out_dir: Path, *, - excludes: "list[str] | None", + excludes: list[str] | None, gitignore: bool | None = None, ) -> None: - """Persist corpus-shaping options under ``out_dir``. - - Best effort and non clobbering: omitted options retain their existing values. - """ + """Persist corpus-shaping scan options used by watch and update.""" if not excludes and gitignore is None: return + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / ".graphify_build.json" try: - out_dir.mkdir(parents=True, exist_ok=True) - path = out_dir / _BUILD_CONFIG_FILENAME - try: - config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} - except (OSError, json.JSONDecodeError): - config = {} - if not isinstance(config, dict): - config = {} - if excludes: - config["excludes"] = list(excludes) - if gitignore is not None: - config["gitignore"] = gitignore - path.write_text(json.dumps(config), encoding="utf-8") - except OSError: - pass + payload = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} + except (OSError, ValueError): + payload = {} + if not isinstance(payload, dict): + payload = {} + if excludes: + payload["excludes"] = list(excludes) + if gitignore is not None: + payload["gitignore"] = gitignore + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") def _read_build_excludes(out_dir: Path) -> list[str]: - """Return the persisted ``--exclude`` patterns for this graph, or [].""" try: - path = out_dir / _BUILD_CONFIG_FILENAME - if path.is_file(): - cfg = json.loads(path.read_text(encoding="utf-8")) - ex = cfg.get("excludes") if isinstance(cfg, dict) else None - if isinstance(ex, list): - return [str(x) for x in ex if isinstance(x, str) and x] - except (OSError, json.JSONDecodeError): - pass - return [] + data = json.loads((out_dir / ".graphify_build.json").read_text(encoding="utf-8")) + values = data.get("excludes", []) + return [str(item) for item in values] if isinstance(values, list) else [] + except (OSError, ValueError): + return [] def _read_build_gitignore(out_dir: Path) -> bool: - """Return whether rebuilds should honor VCS ignore files (default True).""" try: - path = out_dir / _BUILD_CONFIG_FILENAME - if path.is_file(): - cfg = json.loads(path.read_text(encoding="utf-8")) - if isinstance(cfg, dict) and isinstance(cfg.get("gitignore"), bool): - return cfg["gitignore"] - except (OSError, json.JSONDecodeError): - pass - return True - + data = json.loads((out_dir / ".graphify_build.json").read_text(encoding="utf-8")) + value = data.get("gitignore") + return value if isinstance(value, bool) else True + except (OSError, ValueError): + return True -def _merge_changed_paths(*sources: "list[Path] | None") -> list[Path]: - """Concatenate path lists, preserving order and dropping duplicates. - Used to combine a hook process's own ``changed_paths`` with the drained - contents of ``.pending_changes`` so the lock-holding rebuild covers - every queued commit's worth of files (#1059). - """ - seen: set[str] = set() - out: list[Path] = [] - for src in sources: - if not src: - continue - for p in src: - key = os.fspath(p) - if key in seen: - continue - seen.add(key) - out.append(p) - return out +def _stabilize_rebuild_cwd(watch_path: Path) -> bool: + """Recover detached hooks whose inherited working directory was removed.""" + if watch_path.is_absolute(): + return True + repo_root = os.environ.get("GRAPHIFY_REPO_ROOT", "").strip() + if repo_root and Path(repo_root).is_dir(): + try: + os.chdir(repo_root) + return True + except OSError: + pass + try: + Path.cwd() + return True + except FileNotFoundError: + print( + "[graphify watch] Rebuild failed: current working directory no longer " + "exists and GRAPHIFY_REPO_ROOT is not set." + ) + return False -@contextlib.contextmanager +@contextmanager def _rebuild_lock(out_dir: Path, *, blocking: bool = False): - """Per-repo advisory lock around a rebuild. - - Yields True if acquired, False if another rebuild is already running and - ``blocking`` is False. Uses fcntl.flock so the lock is released - automatically if the process is killed (no stale-lock cleanup needed). - - While the lock is held, ``.rebuild.lock`` contains the owning PID followed - by a newline so external pollers (publish scripts, etc.) can read it. - On successful release the file is unlinked so downstream tooling that - waits for the lock to clear by polling for its absence unblocks promptly. - - Falls back to a no-op yield(True) on platforms without fcntl (Windows). - """ - try: - import fcntl - except ImportError: - yield True - return + """Serialize local rebuild preparation; Helix also enforces writer exclusion.""" + from graphify.helix.persistence import _StoreLock out_dir.mkdir(parents=True, exist_ok=True) lock_path = out_dir / ".rebuild.lock" - # "a+" creates the file if missing without truncating an existing holder's - # PID payload — important because another process may have already written - # its PID before we attempt the flock. - fh = open(lock_path, "a+", encoding="utf-8") + lock = _StoreLock( + lock_path, + shared=False, + timeout=120.0 if blocking else 0.0, + ) acquired = False try: - flags = fcntl.LOCK_EX if blocking else (fcntl.LOCK_EX | fcntl.LOCK_NB) try: - fcntl.flock(fh.fileno(), flags) - except BlockingIOError: + lock.acquire() + except TimeoutError: yield False - return - acquired = True - # Replace any prior owner's PID with ours so external readers see a - # single parseable line, not a digit-concatenation across rebuilds. - try: - fh.seek(0) - fh.truncate() - fh.write(f"{os.getpid()}\n") - fh.flush() - except OSError: - pass - yield True + else: + acquired = True + yield True finally: - if acquired: - try: - fcntl.flock(fh.fileno(), fcntl.LOCK_UN) - except OSError: - pass - fh.close() - # Signal "rebuild done" by removing the lock file. Only the holder - # unlinks; a non-acquiring caller leaves the existing lock in place. + lock.release() if acquired: with contextlib.suppress(OSError): lock_path.unlink() -def _apply_resource_limits() -> None: - """Best-effort nice + memory cap. Called from inline hook scripts. - - GRAPHIFY_REBUILD_MEMORY_LIMIT_MB caps RSS-ish memory. Uses RLIMIT_DATA on - macOS (RLIMIT_AS is unreliable under Apple's libmalloc) and RLIMIT_AS on - Linux. Silently skips if the platform doesn't support it. - """ - try: - os.nice(10) - except (OSError, AttributeError): - pass - mb = os.environ.get("GRAPHIFY_REBUILD_MEMORY_LIMIT_MB", "").strip() - if not mb: - return - try: - limit = int(mb) * 1024 * 1024 - except ValueError: - return - try: - import resource - which = resource.RLIMIT_DATA if sys.platform == "darwin" else resource.RLIMIT_AS - soft, hard = resource.getrlimit(which) - new_hard = hard if hard != resource.RLIM_INFINITY and hard < limit else limit - resource.setrlimit(which, (limit, new_hard)) - except (ImportError, ValueError, OSError): - pass - - -def _git_head() -> str | None: - """Return current git HEAD commit hash, or None outside a repo.""" - import subprocess as _sp - try: - r = _sp.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=3) - return r.stdout.strip() if r.returncode == 0 else None - except Exception: - return None - - -from graphify.detect import ( - CODE_EXTENSIONS, - DOC_EXTENSIONS, - PAPER_EXTENSIONS, - IMAGE_EXTENSIONS, - _load_graphifyignore, - _is_ignored, -) - -_WATCHED_EXTENSIONS = CODE_EXTENSIONS | DOC_EXTENSIONS | PAPER_EXTENSIONS | IMAGE_EXTENSIONS -_CODE_EXTENSIONS = CODE_EXTENSIONS - - -def _report_root_label(watch_path: Path) -> str: - if watch_path.is_absolute(): - return watch_path.name or str(watch_path) - return Path.cwd().name if watch_path == Path(".") else str(watch_path) - - -def _is_relative_to(path: Path, root: Path) -> bool: - try: - path.relative_to(root) - return True - except ValueError: - return False - - -def _changed_path_candidates(raw: Path, *, change_root: Path, watch_root: Path) -> list[Path]: - """Return plausible absolute locations for a hook-provided changed path. - - Git hooks pass paths relative to the repository root. Watch callers may - also pass paths relative to the watched root. Keep both interpretations so - a graph rooted at ``src`` accepts ``src/app.py`` and ``app.py``. - """ - if raw.is_absolute(): - lexical = Path(os.path.abspath(raw)) - resolved = raw.resolve() - return [lexical] if lexical == resolved else [lexical, resolved] - - candidates: list[Path] = [] - seen: set[str] = set() - for base in (change_root, watch_root): - lexical = Path(os.path.abspath(base / raw)) - for cand in (lexical, lexical.resolve()): - key = os.fspath(cand) - if key in seen: - continue - seen.add(key) - candidates.append(cand) - return candidates - - -def _relativize_source_files(payload: dict, root: Path, *, scope: Path | None = None) -> None: - for bucket in ("nodes", "edges", "hyperedges"): - for item in payload.get(bucket, []): - source = item.get("source_file") - if not source: - continue - source_path = Path(source) - if not source_path.is_absolute(): - continue - try: - resolved = source_path.resolve() - if scope is not None and not _is_relative_to(resolved, scope): - continue - item["source_file"] = resolved.relative_to(root).as_posix() - except ValueError: - continue - - -def _rebase_relative_source_files(payload: dict, source_root: Path, target_root: Path) -> None: - """Rebase cache-root-relative source paths onto the project root.""" - if source_root == target_root: - return - for bucket in ("nodes", "edges", "hyperedges"): - for item in payload.get(bucket, []): - source = item.get("source_file") - if not source or Path(source).is_absolute(): - continue - try: - item["source_file"] = (source_root / source).relative_to(target_root).as_posix() - except ValueError: - continue - - -class _StoredSourcePaths: - """Resolve source_file values across current and legacy graph roots.""" - - def __init__( - self, - existing: dict, - *, - out: Path, - project_root: Path, - watch_root: Path, - normalize_source, - ) -> None: - self.project_root = project_root - self.watch_root = watch_root - self._normalize_source = normalize_source - self.existing_source_root = project_root - relative_marker_prefix: str | None = None - - root_marker = out / ".graphify_root" - if root_marker.exists(): - try: - saved_root = Path(root_marker.read_text(encoding="utf-8").strip()) - if saved_root.is_absolute(): - self.existing_source_root = saved_root.resolve() - else: - invocation_root = Path.cwd().resolve() - if (invocation_root / saved_root).resolve() == watch_root: - self.existing_source_root = invocation_root - relative_marker_prefix = posixpath.normpath(saved_root.as_posix()) - except (OSError, ValueError): - pass - - self.legacy_watch_relative = False - if relative_marker_prefix not in (None, "."): - has_project_relative_source = False - for bucket in ("nodes", "links", "edges", "hyperedges"): - for item in existing.get(bucket, []): - stored = normalize_source(item.get("source_file")) - if not stored or Path(stored).is_absolute(): - continue - normalized = posixpath.normpath(stored) - if ( - normalized == relative_marker_prefix - or normalized.startswith(relative_marker_prefix + "/") - ): - has_project_relative_source = True - break - if has_project_relative_source: - break - self.legacy_watch_relative = not has_project_relative_source - - def normalize(self, source_file: str | None) -> str | None: - normalized = self._normalize_source(source_file, str(self.project_root)) - return posixpath.normpath(normalized) if normalized else normalized - - def absolute_identity(self, source_file: str | None, root: Path) -> str | None: - normalized = self._normalize_source(source_file) - if not normalized: - return normalized - source_path = Path(posixpath.normpath(normalized)) - if not source_path.is_absolute(): - source_path = root / source_path - return Path(os.path.abspath(source_path)).as_posix() - - def identity(self, source_file: str | None) -> str | None: - normalized = self._normalize_source(source_file) - if normalized and not Path(normalized).is_absolute() and self.legacy_watch_relative: - return self.absolute_identity(normalized, self.watch_root) - return self.absolute_identity(normalized, self.existing_source_root) - - def in_watch_root(self, source_file: str | None) -> bool: - identity = self.identity(source_file) - return bool(identity) and _is_relative_to(Path(identity), self.watch_root) - - def is_evicted(self, item: dict, identities: set[str]) -> bool: - return self.identity(item.get("source_file")) in identities - - def rebase_preserved(self, item: dict) -> None: - identity = self.identity(item.get("source_file")) - if not identity: - return - identity_path = Path(identity) - if not _is_relative_to(identity_path, self.watch_root): - normalized = self.normalize(item.get("source_file")) - if normalized: - item["source_file"] = normalized - return - try: - item["source_file"] = identity_path.relative_to(self.project_root).as_posix() - except ValueError: - item["source_file"] = identity - - -def _reconcile_existing_graph( - existing_graph: Path, - result: dict, - *, - out: Path, - project_root: Path, - watch_root: Path, - code_files: list[Path], - extract_targets: list[Path], - full_rebuild: bool, - deleted_paths: set[str], - deleted_source_identities: set[str], -) -> tuple[dict, dict]: - """Merge fresh extraction with preserved graph entries and evict stale sources.""" - existing_graph_data: dict = {} - if not existing_graph.exists(): - return result, existing_graph_data - - try: - from graphify.build import _norm_source_file as _nsf - from graphify.extract import _get_extractor - from graphify.security import check_graph_file_size_cap - - check_graph_file_size_cap(existing_graph) - existing = json.loads(existing_graph.read_text(encoding="utf-8")) - existing_graph_data = existing - source_paths = _StoredSourcePaths( - existing, - out=out, - project_root=project_root, - watch_root=watch_root, - normalize_source=_nsf, +def _git_head(root: Path) -> str | None: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, + check=False, + ) + value = result.stdout.strip() + return value if result.returncode == 0 and value else None + + +def _community_labels(graph, communities: dict[int, list]) -> dict[int, str]: + labels: dict[int, str] = {} + for community_id, members in communities.items(): + ranked = sorted( + members, + key=lambda node: ( + -graph.degree(node).degree, + str(node_attributes(graph, node).get("label", node)), + ), ) - new_ast_ids = {n["id"] for n in result["nodes"]} - current_sources = { - source_paths.absolute_identity(str(path), project_root) for path in code_files - } - rebuilt_source_identities = { - source_paths.absolute_identity(str(path), project_root) for path in extract_targets - } - node_evicted_source_identities = set(deleted_source_identities) - hyperedge_evicted_source_identities = set(deleted_source_identities) - # Deletion evicts edges regardless of tier; re-extraction only owns a - # source's AST-tier edges (checked per-edge below, #1865). - edge_evicted_source_identities = set(deleted_source_identities) - if not full_rebuild: - node_evicted_source_identities.update(rebuilt_source_identities) - - # Reconcile every rebuild against the current watched corpus. Hook change - # lists can contain only a rename destination, so explicit paths alone - # cannot identify the stale source. Keep the comparison scoped to the - # watched root so subfolder updates preserve records outside that subtree. - # - # Fail-closed eviction: a source identity missing from the corpus is only - # DELETION evidence when the file is actually gone from disk. A file that - # still exists but stopped being collected was *excluded* (ignore rules or - # filters changed — e.g. a .gitignore the scanner newly honors), and - # treating that as deletion silently mass-evicts good nodes. Preserve - # instead and say so; a full re-extraction still purges deliberately - # excluded sources via the AST ownership rule below. - excluded_alive_files: set[str] = set() - excluded_alive_nodes = 0 - _alive_cache: dict[str, bool] = {} - for node in existing.get("nodes", []): - source_file = node.get("source_file") - if not source_file or _get_extractor(Path(source_file)) is None: - continue - identity = source_paths.identity(source_file) - if not source_paths.in_watch_root(source_file): - continue - if identity not in current_sources: - if identity: - alive = _alive_cache.get(identity) - if alive is None: - alive = Path(identity).exists() - _alive_cache[identity] = alive - if alive: - excluded_alive_files.add(identity) - excluded_alive_nodes += 1 - continue - normalized = source_paths.normalize(source_file) - if normalized: - deleted_paths.add(normalized) - if identity: - node_evicted_source_identities.add(identity) - edge_evicted_source_identities.add(identity) - hyperedge_evicted_source_identities.add(identity) - if excluded_alive_files: - print( - f"[graphify watch] fail-closed: kept {excluded_alive_nodes} node(s) " - f"from {len(excluded_alive_files)} file(s) that left the scan corpus " - "but still exist on disk (ignore rules or filters changed?). " - "Run a full re-extraction to purge them if the exclusion is intentional." - ) - - # A full re-extraction owns every AST node under watch_root. Incremental - # extraction owns only nodes from rebuilt or deleted sources. Semantic - # nodes lack the AST origin marker and remain preserved. - preserved_nodes = [ - node - for node in existing.get("nodes", []) - if node["id"] not in new_ast_ids - and not ( - node.get("_origin") == "ast" - and ( - ( - not node.get("source_file") - and (full_rebuild or not code_files) - ) - or ( - full_rebuild - and source_paths.in_watch_root(node.get("source_file")) - ) - ) - ) - and not source_paths.is_evicted(node, node_evicted_source_identities) + names = [ + str(node_attributes(graph, node).get("label", node)) + for node in ranked[:2] ] - all_ids = new_ast_ids | {node["id"] for node in preserved_nodes} - - # Edges are owned by source_file, but ownership is tier-scoped: the AST - # pass replaces a re-extracted source's AST edges, while that source's - # semantic/LLM edges — which the AST pass cannot regenerate — survive - # until a semantic re-extraction supersedes them. Same provenance rule - # the node reconciliation above applies via _origin (#1865). Deletion - # eviction stays provenance-blind. - preserved_edges = [ - edge - for edge in existing.get("links", existing.get("edges", [])) - if edge.get("source") in all_ids - and edge.get("target") in all_ids - and not source_paths.is_evicted(edge, edge_evicted_source_identities) - and not ( - edge.get("_origin") == "ast" - and source_paths.is_evicted(edge, rebuilt_source_identities) - ) - ] - - new_hyperedge_ids = { - edge.get("id") for edge in result.get("hyperedges", []) if edge.get("id") - } - preserved_hyperedges = [] - for edge in existing.get("hyperedges", []): - members = edge.get("nodes", edge.get("members", edge.get("node_ids", []))) - if edge.get("id") in new_hyperedge_ids or source_paths.is_evicted( - edge, hyperedge_evicted_source_identities - ): - continue - if isinstance(members, list) and any(member not in all_ids for member in members): - continue - preserved_hyperedges.append(edge) + labels[community_id] = " & ".join(names) if names else f"Community {community_id}" + return labels - for item in preserved_nodes + preserved_edges + preserved_hyperedges: - source_paths.rebase_preserved(item) - return { - "nodes": result["nodes"] + preserved_nodes, - "edges": result["edges"] + preserved_edges, - "hyperedges": result.get("hyperedges", []) + preserved_hyperedges, - "input_tokens": 0, - "output_tokens": 0, - }, existing_graph_data - except Exception: - return result, existing_graph_data - - -def _node_community_map(graph_data: dict) -> dict[str, int]: - out: dict[str, int] = {} - for node in graph_data.get("nodes", []): - node_id = node.get("id") - cid = node.get("community") - if node_id is None or cid is None: - continue - try: - out[str(node_id)] = int(cid) - except (TypeError, ValueError): - print( - f"[graphify watch] Skipping node with invalid community id: " - f"node_id={node_id!r} community={cid!r}", - file=sys.stderr, - ) - continue - return out - - -def _canonical_graph_for_compare(graph_data: dict) -> dict: - canonical = dict(graph_data) - canonical.pop("built_at_commit", None) - for key in ("nodes", "links", "edges", "hyperedges"): - if key in canonical and isinstance(canonical[key], list): - canonical[key] = sorted( - canonical[key], - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), - ) - return canonical +def _content_hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() -def _canonical_topology_for_compare(graph_data: dict) -> dict: - canonical = dict(graph_data) - canonical.pop("built_at_commit", None) - - nodes = canonical.get("nodes") - if isinstance(nodes, list): - norm_nodes = [] - for node in nodes: - if not isinstance(node, dict): - continue - n = dict(node) - n.pop("community", None) - n.pop("community_name", None) - n.pop("norm_label", None) - norm_nodes.append(n) - canonical["nodes"] = sorted( - norm_nodes, - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), - ) - - for key in ("links", "edges"): - items = canonical.get(key) - if not isinstance(items, list): - continue - norm_edges = [] - for edge in items: - if not isinstance(edge, dict): - continue - e = dict(edge) - # to_json writes _src/_tgt as the canonical directed endpoints and - # overwrites source/target with them before serialising, so the - # on-disk graph has no _src/_tgt. The candidate topology (fresh from - # node_link_data) still has them. Popping and reassigning here makes - # both sides comparable: existing gets no-op pops (None), candidate - # gets source/target overwritten from _src/_tgt — same result. - true_src = e.pop("_src", None) - true_tgt = e.pop("_tgt", None) - if true_src is not None and true_tgt is not None: - e["source"] = true_src - e["target"] = true_tgt - e.pop("confidence_score", None) - norm_edges.append(e) - canonical[key] = sorted( - norm_edges, - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), - ) - - hyperedges = canonical.get("hyperedges") - if isinstance(hyperedges, list): - canonical["hyperedges"] = sorted( - hyperedges, - key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False, default=str), +def _warn_obsolete_store(out: Path) -> None: + legacy = out / "graph.json" + if legacy.exists(): + print( + "[graphify] warning: graph.json is obsolete and ignored; rebuild from source to create graph.helix.", + file=sys.stderr, ) - return canonical - - -def _topology_from_graph(G) -> dict: - from networkx.readwrite import json_graph - try: - data = json_graph.node_link_data(G, edges="links") - except TypeError: - data = json_graph.node_link_data(G) - data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - return data - def _check_shrink( force: bool, existing_data: dict, new_data: dict, - tmp: "Path | None" = None, + tmp: Path | None = None, *, had_explicit_deletions: bool = False, - rebuilt_sources: "set[str] | None" = None, + rebuilt_sources: set[str] | None = None, ) -> bool: - """Return True (ok to proceed) or False (shrink refused). - - When False, cleans up *tmp* if provided and prints a warning to stderr. - - The shrink-guard exists to catch SILENT shrinkage from failed extraction - chunks (a half-written semantic pass leaving thousands of nodes - unaccounted for). When ``had_explicit_deletions`` is True, the caller - has declared which files were removed (e.g. the post-commit hook saw - a ``D`` in ``git diff --name-only``) and a smaller graph is the expected - outcome — skip the guard so legitimate refactors don't require ``--force``. - - ``rebuilt_sources`` (when given) is the set of source files re-extracted this - run. A net shrink is legitimate — not a failed chunk — when every *lost* node - belonged to one of those files (a symbol removed from a re-extracted file) or - carries no source_file. Only an unexplained loss (a node from a file we did - NOT touch — e.g. a dropped semantic/doc node) refuses the write. This lets a - plain ``graphify update`` after deleting a function refresh the graph without - ``--force`` (#1116 left stale nodes write-blocked even though build dropped them). - """ + """Refuse unexplained node loss before activating a staged generation.""" if force or not existing_data or had_explicit_deletions: return True existing_nodes = existing_data.get("nodes", []) @@ -729,641 +252,785 @@ def _check_shrink( return True if rebuilt_sources is not None: from graphify.build import _norm_source_file - new_ids = {n.get("id") for n in new_nodes} - lost = [n for n in existing_nodes if n.get("id") not in new_ids] - - def _accounted(n: dict) -> bool: - sf = n.get("source_file") - return (not sf - or sf in rebuilt_sources - or _norm_source_file(sf) in rebuilt_sources) - if all(_accounted(n) for n in lost): + + new_ids = {node.get("id") for node in new_nodes} + lost = [node for node in existing_nodes if node.get("id") not in new_ids] + + def accounted(node: dict) -> bool: + source = node.get("source_file") + return bool( + not source + or source in rebuilt_sources + or _norm_source_file(source) in rebuilt_sources + ) + + if all(accounted(node) for node in lost): return True if tmp is not None: - tmp.unlink(missing_ok=True) + if tmp.is_dir(): + import shutil + + shutil.rmtree(tmp) + else: + tmp.unlink(missing_ok=True) print( - f"[graphify] WARNING: new graph has {len(new_nodes)} nodes but existing " - f"graph.json has {len(existing_nodes)}. Refusing to overwrite — you may be " - f"missing chunk files from a previous session. " - f"Pass --force to override.", + f"[graphify] WARNING: new graph has {len(new_nodes)} nodes but the active " + f"graph.helix generation has {len(existing_nodes)}. Refusing to overwrite " + "because untouched source nodes disappeared; pass --force to override.", file=sys.stderr, ) return False -def _report_for_compare(report_text: str) -> str: - return re.sub(r"^- Built from commit: `[^`]+`\n?", "", report_text, flags=re.MULTILINE) - - -def _json_text(data: dict) -> str: - return json.dumps(data, indent=2, ensure_ascii=False) + "\n" - - -def _stabilize_rebuild_cwd(watch_path: Path) -> bool: - """Ensure relative rebuild paths have a usable CWD before queue/lock setup. - - Detached git hooks can inherit a transient working directory that is deleted - before the background rebuild starts. In that state Path.cwd(), - Path('.').resolve(), and relative graphify-out mkdirs raise FileNotFoundError - before the normal rebuild error handling can run. Hooks that know the repo - root export GRAPHIFY_REPO_ROOT so the rebuild can recover by chdir'ing there. - """ - if watch_path.is_absolute(): - return True - - repo_root = os.environ.get("GRAPHIFY_REPO_ROOT", "").strip() - if repo_root and Path(repo_root).is_dir(): - try: - os.chdir(repo_root) - return True - except OSError: - pass - - try: - Path.cwd() - return True - except FileNotFoundError: - print( - "[graphify watch] Rebuild failed: current working directory " - "no longer exists and GRAPHIFY_REPO_ROOT is not set." - ) - return False - - def _rebuild_code( watch_path: Path, *, + output_root: Path | None = None, changed_paths: list[Path] | None = None, follow_symlinks: bool = False, force: bool = False, no_cluster: bool = False, acquire_lock: bool = True, block_on_lock: bool = False, + include_semantic: bool = False, + code_only: bool = False, + backend: str | None = None, + model: str | None = None, + deep_mode: bool = False, + token_budget: int = 60_000, + max_concurrency: int = 4, + retain_rollback: bool = False, + raise_on_error: bool = False, + _root_marker: str | None = None, ) -> bool: - """Re-run AST extraction + build + optional cluster + report for code files. No LLM needed. - - When ``force`` is True the node-count safety check in ``to_json`` is bypassed - so the rebuilt graph overwrites graph.json even if it has fewer nodes. - Use this after refactors that legitimately delete code. + """Extract source and atomically activate topology, analysis, and state. - When ``changed_paths`` is provided, only those files are re-extracted; nodes - for unchanged files are preserved from the existing graph. Deleted paths - in ``changed_paths`` (paths that no longer exist on disk) are dropped from - the preserved set. When ``changed_paths`` is None the full code corpus is - re-extracted (used by the watcher and post-checkout hook). - - ``acquire_lock`` (default True) takes a non-blocking per-repo flock around - the rebuild so concurrent post-commit hooks across multiple repos do not - pile up. Returns False with a log line if the lock is held. Pass - ``block_on_lock=True`` to wait instead of skip (used by the interactive - ``graphify update`` CLI). - - ``no_cluster`` skips community detection and writes raw merged extraction - JSON to graphify-out/graph.json (mirrors ``extract --no-cluster``). - - Returns True on success, False on error or skipped-due-to-lock. + ``update`` and git hooks leave ``include_semantic`` false, so they remain + local AST-only refreshes. The headless ``extract`` command enables it and + feeds documents, papers, and images through the configured LLM backend. """ + watch_path = Path(watch_path) + if _root_marker is None: + _root_marker = os.fspath(watch_path) if not _stabilize_rebuild_cwd(watch_path): return False - - out = watch_path / _GRAPHIFY_OUT + watch_path = watch_path.resolve() + source_root = ( + watch_path + if Path(_root_marker).is_absolute() + else Path.cwd().resolve() + ) + output_root = Path(output_root).resolve() if output_root is not None else watch_path + out = output_root / _GRAPHIFY_OUT if acquire_lock: - # #1059: incremental (changed_paths is not None) hooks must not drop - # their change set when another rebuild is already running. Queue - # before attempting the lock so a non-blocking failure still records - # the work; the lock-holder drains the queue and merges it in. Full- - # corpus rebuilds skip the queue entirely — they already cover every - # file, so there is nothing to merge. - if changed_paths is not None and not block_on_lock: - _queue_pending(out, list(changed_paths)) - with _rebuild_lock(out, blocking=block_on_lock) as got: - if not got: - print("[graphify watch] Rebuild already in progress for " - f"{watch_path.resolve()} - changes queued.") + with _rebuild_lock(out, blocking=block_on_lock) as acquired: + if not acquired: + if changed_paths is not None: + _queue_pending(out, changed_paths) + print( + f"[graphify watch] Rebuild already in progress for {watch_path}; " + f"queued {len(changed_paths)} changed path(s)." + ) + else: + print(f"[graphify watch] Rebuild already in progress for {watch_path}.") return False - # Lock acquired. Drain anything queued by earlier contenders - # (including, importantly, the paths we just queued ourselves) - # and merge with our own change set so a single rebuild covers - # everything outstanding. - if changed_paths is not None: - merged = _merge_changed_paths(changed_paths, _drain_pending(out)) - else: - # Full-corpus rebuild supersedes any queued incremental work. - _drain_pending(out) - merged = None - ok = _rebuild_code( - watch_path, - changed_paths=merged, - follow_symlinks=follow_symlinks, - force=force, - no_cluster=no_cluster, - acquire_lock=False, + + queued = _drain_pending(out) + first_paths = ( + None if changed_paths is None else _merge_changed_paths(changed_paths, queued) ) - # Late-arrival drain: another hook may have queued work while we - # were rebuilding. Loop up to _PENDING_DRAIN_MAX_PASSES times so a - # storm of commits eventually quiesces without livelocking. A full - # rebuild already saw everything, so skip this for changed_paths is None. - if merged is not None: - for _ in range(_PENDING_DRAIN_MAX_PASSES): - late = _drain_pending(out) - if not late: - break - ok = _rebuild_code( - watch_path, - changed_paths=late, - follow_symlinks=follow_symlinks, - force=force, - no_cluster=no_cluster, - acquire_lock=False, - ) and ok - return ok - - watch_root = watch_path.resolve() - project_root = Path.cwd().resolve() if not watch_path.is_absolute() else watch_root - report_root = _report_root_label(watch_path) + + def run_inner(paths: list[Path] | None) -> bool: + return _rebuild_code( + watch_path, + changed_paths=paths, + output_root=output_root, + follow_symlinks=follow_symlinks, + force=force, + no_cluster=no_cluster, + acquire_lock=False, + include_semantic=include_semantic, + code_only=code_only, + backend=backend, + model=model, + deep_mode=deep_mode, + token_budget=token_budget, + max_concurrency=max_concurrency, + retain_rollback=retain_rollback, + raise_on_error=raise_on_error, + _root_marker=_root_marker, + ) + + result = run_inner(first_paths) + if changed_paths is None: + return result + for _ in range(_PENDING_DRAIN_MAX_PASSES): + late = _drain_pending(out) + if not late: + break + result = run_inner(late) and result + return result + try: - from graphify.extract import extract, _get_extractor + from graphify.analyze import god_nodes, suggest_questions, surprising_connections + from graphify.build import build_from_extraction, build_unclustered_extraction + from graphify.cluster import cluster, score_all + from graphify.cache import check_semantic_cache, save_semantic_cache from graphify.detect import detect - from graphify.build import build_from_json, _norm_source_file as _nsf - from graphify.cluster import cluster, remap_communities_to_previous, score_all - from graphify.analyze import god_nodes, surprising_connections, suggest_questions + from graphify.export import to_html + from graphify.extract import extract + from graphify.helix.model import GraphBuildData + from graphify.helix.native import HELIX_PYTHON_VERSION + from graphify.helix.persistence import HelixEmbeddedStore + from graphify.helix.state import ( + communities_from_state, + community_records, + community_summaries, + labels_from_state, + new_state, + ) from graphify.report import generate - from graphify.export import to_json, to_html - from graphify.security import check_graph_file_size_cap - # Re-apply the excludes the initial extract recorded, so an update/watch/ - # hook rebuild does not silently re-include deliberately excluded paths - # (#1886). - _persisted_excludes = _read_build_excludes(out) + out.mkdir(parents=True, exist_ok=True) + _warn_obsolete_store(out) + store_path = out / "graph.helix" detected = detect( - watch_path, follow_symlinks=follow_symlinks, - extra_excludes=_persisted_excludes or None, + watch_path, + follow_symlinks=follow_symlinks, + extra_excludes=_read_build_excludes(out) or None, gitignore=_read_build_gitignore(out), ) - code_files = [Path(f) for f in detected['files']['code']] - - # Include document files that have AST extractors (e.g. .md, .mdx, .qmd) - ast_doc_files: list[Path] = [] - for doc_file in detected['files'].get('document', []): - p = Path(doc_file) - if _get_extractor(p) is not None: - code_files.append(p) - ast_doc_files.append(p) - - existing_graph = out / "graph.json" - if not code_files and not existing_graph.exists(): - print("[graphify watch] No code files found - nothing to rebuild.") - return False + files_by_type = detected.get("files", {}) + code_files = [Path(item) for item in files_by_type.get("code", [])] + semantic_files = [ + Path(item) + for kind in ("document", "paper", "image") + for item in files_by_type.get(kind, []) + ] + quick_document_files = [ + path + for path in semantic_files + if path.suffix.lower() in {".md", ".mdx", ".qmd", ".skill"} + ] + quick_files = [*code_files, *quick_document_files] + if deep_mode: + semantic_files = [*code_files, *semantic_files] + if code_only: + semantic_files = [] + quick_files = code_files + current_files = [*code_files, *semantic_files] + current_absolute = { + identity + for path in current_files + for identity in ( + Path(os.path.abspath(path)), + path.resolve(), + ) + } - # #1915: a document that already carries SEMANTIC (LLM) nodes in the - # existing graph must not ALSO be AST-quick-scanned — otherwise every - # rebuild mints heading nodes on top of the preserved semantic nodes - # and the doc is represented twice (~4x graph bloat vs the CLI update - # path, which AST-extracts only code). Semantic supersedes AST per doc - # source: the quick-scan stays as a fallback for docs with no semantic - # layer (the no-LLM doc-structure feature, #09b33b7) and for brand-new - # docs the graph has never seen. These docs stay in ``code_files`` so - # corpus membership (#1795 fail-closed deletion evidence) and the - # shrink accounting below still cover them — a previously-bloated - # graph must be allowed to self-heal on a full rebuild without the - # shrink-guard refusing the smaller write. - semantic_doc_files: set[Path] = set() - if ast_doc_files and existing_graph.exists(): + def resolve_changed(path: Path) -> Path: + if path.is_absolute(): + return Path(os.path.abspath(path)) + candidates = [ + Path(os.path.abspath(source_root / path)), + Path(os.path.abspath(watch_path / path)), + ] + for candidate in candidates: + if candidate in current_absolute: + return candidate + for candidate in candidates: + if candidate.exists(): + return candidate + return candidates[0] if candidates[0].is_relative_to(watch_path) else candidates[1] + + deleted_sources: set[str] = set() + if changed_paths is not None: + requested = [resolve_changed(Path(path)) for path in changed_paths] + extract_targets = [ + path for path in requested + if path.resolve() in {item.resolve() for item in quick_files} and path.is_file() + ] + semantic_targets = [ + path for path in requested + if include_semantic + and path.resolve() in {item.resolve() for item in semantic_files} + and path.is_file() + ] + deleted_sources.update( + path.relative_to(source_root).as_posix() + for path in requested + if not path.exists() + and path.is_relative_to(watch_path) + and path.is_relative_to(source_root) + ) + else: + extract_targets = quick_files + semantic_targets = semantic_files if include_semantic else [] + + previous_state = new_state() + loaded = None + had_existing_generation = False + existing_source_root = source_root + marker = out / ".graphify_root" + saved_root = Path(".") + if marker.is_file(): try: - check_graph_file_size_cap(existing_graph) - prior = json.loads(existing_graph.read_text(encoding="utf-8")) - prior_paths = _StoredSourcePaths( - prior, - out=out, - project_root=project_root, - watch_root=watch_root, - normalize_source=_nsf, + saved_root = Path(marker.read_text(encoding="utf-8").strip()) + existing_source_root = ( + saved_root.resolve() + if saved_root.is_absolute() + else source_root ) - # Semantic doc nodes lack the AST origin marker. Gate on the - # doc-shaped subset of the six-value file_type enum - # (document/concept/rationale/paper, matching build.py's - # canonical set minus code/image) rather than "document" - # alone: per the extraction spec, a doc full of named - # concepts may be represented with ONLY concept/rationale - # nodes and no separate "document" node — that's still - # evidence of a semantic layer, not a marker-less AST node - # (#1954). The narrower pre-#1954 check under-recognized - # exactly that doc shape, letting it be re-quick-scanned - # every rebuild. A pre-#1865 graph whose AST nodes lack the - # ``_origin`` marker still isn't misread as semantic-backed, - # since "code" stays outside this set. - semantic_doc_identities: set[str] = set() - for node in prior.get("nodes", []): - if node.get("_origin") == "ast": - continue - if node.get("file_type") not in ( - "document", "concept", "rationale", "paper" + except (OSError, ValueError): + pass + if store_path.is_dir(): + try: + # Updates are already a writer operation. Open through the + # ordinary public writer so Helix can complete any required + # embedded-format migration before Graphify takes a snapshot. + with HelixEmbeddedStore(store_path) as existing_store: + if no_cluster: + previous_state = copy.deepcopy(existing_store.read_state()) + had_existing_generation = True + else: + loaded = existing_store.load() + previous_state = copy.deepcopy(dict(loaded.state)) + had_existing_generation = True + except RuntimeError as exc: + if "no active generation" not in str(exc): + raise + if had_existing_generation: + stored_files = previous_state.get("incremental", {}).get("files", {}) + stored_sources = ( + [str(source) for source in stored_files] + if isinstance(stored_files, dict) + else [] + ) + if marker.is_file() and not saved_root.is_absolute() and saved_root != Path("."): + marker_prefix = saved_root.as_posix().rstrip("/") + "/" + if stored_sources and not any( + source.startswith(marker_prefix) for source in stored_sources ): + existing_source_root = watch_path + excluded_alive: set[str] = set() + for source in stored_sources: + source_path = Path(str(source)) + absolute = ( + Path(os.path.abspath(source_path)) + if source_path.is_absolute() + else Path(os.path.abspath(existing_source_root / source_path)) + ) + if not absolute.is_relative_to(watch_path): continue - identity = prior_paths.identity(node.get("source_file")) - if identity: - semantic_doc_identities.add(identity) - if semantic_doc_identities: - semantic_doc_files = { - p for p in ast_doc_files - if prior_paths.absolute_identity(str(p), project_root) - in semantic_doc_identities - } - except Exception: - semantic_doc_files = set() - - # Incremental path: when the caller passed an explicit change list, - # extract only changed-and-still-existing files. Deleted paths are - # tracked separately so their stale nodes can be evicted below. - deleted_paths: set[str] = set() - deleted_source_identities: set[str] = set() - def _add_deleted_source(path: Path) -> None: - deleted_source_identities.add(Path(os.path.abspath(path)).as_posix()) - for root in (project_root, watch_root): - deleted_paths.add(_nsf(str(path), str(root)) or str(path)) + if not absolute.exists(): + deleted_sources.add(str(source).replace("\\", "/")) + elif absolute not in current_absolute: + excluded_alive.add(str(source)) + if excluded_alive: + print( + "[graphify watch] fail-closed: kept native nodes from " + f"{len(excluded_alive)} excluded-but-existing source file(s)." + ) - if changed_paths is not None: - code_set = {Path(os.path.abspath(p)) for p in code_files} - # #1915: semantic-backed docs are never AST-quick-scanned; their - # semantic nodes are the sole representation. Mirroring #1865's - # tier-scoped edge rule at the node level, they also must NOT - # enter extract_targets (hence rebuilt/node-evicted identities) on - # an incremental rebuild, or their semantic nodes would be wiped. - semantic_doc_set = {Path(os.path.abspath(p)) for p in semantic_doc_files} - wanted: list[Path] = [] - change_root = Path.cwd().resolve() - for raw in changed_paths: - candidates = _changed_path_candidates( - raw, - change_root=change_root, - watch_root=watch_root, - ) - tracked = next((cand for cand in candidates if cand.exists() and cand in code_set), None) - if tracked is not None: - if tracked not in wanted and tracked not in semantic_doc_set: - wanted.append(tracked) + semantic_backed_sources: set[str] = set() + previous_cache = previous_state.get("incremental", {}).get( + "extraction_cache", {} + ) + if isinstance(previous_cache, dict): + for entry in previous_cache.values(): + if not isinstance(entry, dict) or not str(entry.get("kind", "")).startswith( + "semantic" + ): continue - - existing_in_root = next( - ( - cand for cand in candidates - if cand.exists() and _is_relative_to(cand, watch_root) - ), - None, - ) - if existing_in_root is not None: - # The path exists under the watched root but detect filtered - # it out. Evict any stale nodes that still claim it. - _add_deleted_source(existing_in_root) + result_value = entry.get("result", {}) + if not isinstance(result_value, dict): + continue + for node in result_value.get("nodes", []): + if not isinstance(node, dict): + continue + source = node.get("source_file") + if source and Path(str(source)).suffix.lower() in { + ".md", ".mdx", ".qmd", ".skill", + }: + semantic_backed_sources.add(str(source).replace("\\", "/")) + if semantic_backed_sources: + extract_targets = [ + path + for path in extract_targets + if Path(os.path.abspath(path)).relative_to(source_root).as_posix() + not in semantic_backed_sources + ] + + cache_state = copy.deepcopy( + previous_state.get("incremental", {}).get("extraction_cache", {}) + ) + if not isinstance(cache_state, dict): + cache_state = {} + if force or os.environ.get("GRAPHIFY_FORCE", "").strip() == "1": + cache_state.clear() + if deleted_sources: + deleted_suffixes = {":" + source.replace("\\", "/") for source in deleted_sources} + for key in list(cache_state): + if any(key.endswith(suffix) for suffix in deleted_suffixes): + del cache_state[key] + + build_root = existing_source_root if had_existing_generation else source_root + + def cached_source_paths(kind_prefix: str) -> list[Path]: + paths: dict[str, Path] = {} + for entry in cache_state.values(): + if not isinstance(entry, dict) or not str(entry.get("kind", "")).startswith( + kind_prefix + ): continue + cached_result = entry.get("result", {}) + if not isinstance(cached_result, dict): + continue + for bucket in ("nodes", "edges", "hyperedges"): + for item in cached_result.get(bucket, []): + if not isinstance(item, dict) or not item.get("source_file"): + continue + source_path = Path(str(item["source_file"])) + unresolved = ( + source_path + if source_path.is_absolute() + else build_root / source_path + ) + absolute = Path(os.path.abspath(unresolved)) + if absolute.is_file(): + paths[os.fspath(absolute)] = absolute + return list(paths.values()) + + if not extract_targets and not semantic_targets and not deleted_sources and not store_path.is_dir(): + print("[graphify watch] No supported source files found - nothing to rebuild.") + return False - deleted_in_root = next( - (cand for cand in candidates if _is_relative_to(cand, watch_root)), - None, - ) - if deleted_in_root is not None: - # File was deleted or renamed away inside the watched root. - # Evict preserved nodes that still claim this source path. - _add_deleted_source(deleted_in_root) - if not wanted and not deleted_paths: - print("[graphify watch] No tracked code files in change set - skipping rebuild.") - return True - extract_targets = wanted - else: - # Full rebuild: skip the AST quick-scan for semantic-backed docs - # (#1915). They remain in code_files, so stale _origin=="ast" - # heading nodes from a previously-bloated graph are dropped by the - # full-rebuild AST ownership rule while the shrink accounting - # below still counts the doc as a rebuilt source. - extract_targets = [p for p in code_files if p not in semantic_doc_files] - - commit = _git_head() - result = extract(extract_targets, cache_root=watch_root) if extract_targets else { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, + # Build from extraction DTOs, never by projecting the active native + # topology back into Python. ``extract`` receives the complete live file + # set; unchanged files are served from the cache stored in Helix state, + # while only invalidated files run an extractor. + ast_candidates = { + os.fspath(Path(os.path.abspath(path))): Path(os.path.abspath(path)) + for path in [*quick_files, *cached_source_paths("ast")] + if Path(path).is_file() } - _rebase_relative_source_files(result, watch_root, project_root) - - # Preserve semantic nodes/edges from a previous full run. - # AST-only rebuild replaces nodes for changed files; everything else is kept. - # Filter by node ID membership in the new AST output, not by file_type — - # INFERRED/AMBIGUOUS nodes extracted from code files also carry file_type="code" - # and would be wrongly dropped by a file_type-based filter. - # When the caller supplied changed_paths, also evict preserved nodes whose - # source_file matches a path that was changed (re-extracted) or deleted — - # otherwise the old nodes for those files would survive forever. - result, existing_graph_data = _reconcile_existing_graph( - existing_graph, - result, - out=out, - project_root=project_root, - watch_root=watch_root, - code_files=code_files, - extract_targets=extract_targets, - full_rebuild=changed_paths is None, - deleted_paths=deleted_paths, - deleted_source_identities=deleted_source_identities, + ast_build_files = [] + for path in ast_candidates.values(): + try: + relative = path.relative_to(build_root).as_posix() + except ValueError: + continue + if relative not in semantic_backed_sources: + ast_build_files.append(path) + ast_result = extract( + ast_build_files, root=build_root, cache=cache_state + ) if ast_build_files else { + "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, + } + selected_backend = backend + semantic_result = { + "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, + "output_tokens": 0, + } + previous_semantic = previous_state.get("semantic", {}) + semantic_was_used = ( + isinstance(previous_semantic, dict) + and bool(previous_semantic.get("used")) ) - - _relativize_source_files(result, project_root, scope=watch_root) - # Source files re-extracted this run — their symbol sets may legitimately - # shrink (a removed function), so the shrink-guard should not block the - # write when every lost node belongs to one of them (or a deleted file). - _rebuilt_root = str(project_root) - if changed_paths is None: - rebuilt_sources = { - _nsf(str(p.relative_to(project_root)), _rebuilt_root) - for p in code_files if p.is_relative_to(project_root) + semantic_build_files = ( + list({ + os.fspath(Path(os.path.abspath(path))): Path(os.path.abspath(path)) + for path in [*semantic_files, *cached_source_paths("semantic")] + if Path(path).is_file() + }.values()) + if include_semantic or semantic_was_used + else [] + ) + if semantic_build_files: + from graphify.llm import _extraction_system, detect_backend, extract_corpus_parallel + + previous_mode = previous_state.get("incremental", {}).get( + "extractor_state", {} + ).get("mode") + effective_deep = deep_mode or (not include_semantic and previous_mode == "deep") + semantic_mode = "deep" if effective_deep else None + prompt = _extraction_system(deep=effective_deep) + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( + [str(path) for path in semantic_build_files], + cache_state, + root=build_root, + mode=semantic_mode, + prompt=prompt, + allow_stale=not include_semantic, + ) + requested_semantic = { + Path(os.path.abspath(path)) for path in semantic_targets } - else: - rebuilt_sources = {(_nsf(str(p), _rebuilt_root) or str(p)) for p in extract_targets} - rebuilt_sources |= set(deleted_paths) - out.mkdir(exist_ok=True) - - if no_cluster: - # Normalise to "links" key so schema is consistent with the full clustered path. - # Dedupe parallel edges (the clustered path's DiGraph collapses them implicitly); - # without it, --no-cluster + repeated `update` accumulate duplicates and edge - # counts diverge across build modes (#1317). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - candidate_graph_data = { - **{k: v for k, v in result.items() if k not in ("edges", "nodes")}, - "nodes": _dedupe_nodes(result.get("nodes", [])), - "links": _dedupe_edges(result.get("edges", [])), + uncached_targets = [ + Path(path) + for path in uncached + if include_semantic and Path(os.path.abspath(path)) in requested_semantic + ] + fresh = { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, } - candidate_graph_text = _json_text(candidate_graph_data) - same_graph = False - if existing_graph.exists(): - try: - check_graph_file_size_cap(existing_graph) - existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) - same_graph = ( - json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False) - == json.dumps(_canonical_graph_for_compare(candidate_graph_data), sort_keys=True, ensure_ascii=False) + if uncached_targets: + selected_backend = backend or detect_backend() + if selected_backend is None: + raise RuntimeError( + "semantic sources were detected but no LLM backend is configured; " + "pass --backend, configure an API key, or use --code-only" ) - except Exception: - same_graph = False - if not same_graph: - if not _check_shrink( - force, existing_graph_data, candidate_graph_data, - had_explicit_deletions=bool(deleted_paths), - rebuilt_sources=rebuilt_sources, - ): - return False - existing_graph.write_text(candidate_graph_text, encoding="utf-8") - - # Write the user-supplied path only after the candidate graph is - # accepted, so a refused shrink cannot mismatch graph and marker. - (out / ".graphify_root").write_text(str(watch_path), encoding="utf-8") - - try: - from graphify.detect import save_manifest - # detected["files"] is a FULL detect of the watched root, so - # pass it as the scan corpus too: rows for files that left the - # scan but still exist on disk (newly excluded) are pruned - # instead of surviving as phantom "deleted" entries (#1908). - save_manifest( - detected["files"], kind="ast", root=project_root, - scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + fresh = extract_corpus_parallel( + uncached_targets, + backend=selected_backend, + model=model, + root=build_root, + token_budget=token_budget, + max_concurrency=max_concurrency, + deep_mode=effective_deep, + cache=cache_state, + cache_root=output_root, ) - except Exception: - pass - - # clear stale needs_update flag if present - flag = out / "needs_update" - if flag.exists(): - flag.unlink() - - if same_graph: - print("[graphify watch] No code-graph changes detected (--no-cluster); outputs left untouched.") - else: + if fresh.get("failed_chunks"): + raise RuntimeError( + f"semantic extraction failed for {fresh['failed_chunks']} chunk(s); " + "the active generation was left unchanged" + ) + save_semantic_cache( + fresh.get("nodes", []), + fresh.get("edges", []), + fresh.get("hyperedges", []), + root=build_root, + cache_root=output_root, + allowed_source_files=uncached_targets, + mode=semantic_mode, + prompt=prompt, + cache=cache_state, + ) + elif not uncached: + selected_backend = ( + backend or previous_state.get("semantic", {}).get("backend") + ) + semantic_result = { + "nodes": [*cached_nodes, *fresh.get("nodes", [])], + "edges": [*cached_edges, *fresh.get("edges", [])], + "hyperedges": [*cached_hyperedges, *fresh.get("hyperedges", [])], + "input_tokens": fresh.get("input_tokens", 0), + "output_tokens": fresh.get("output_tokens", 0), + } + current_files = list({ + os.fspath(Path(os.path.abspath(path))): Path(os.path.abspath(path)) + for path in [*current_files, *ast_build_files, *semantic_build_files] + if Path(path).is_file() + }.values()) + result = { + "nodes": [*ast_result.get("nodes", []), *semantic_result.get("nodes", [])], + "edges": [*ast_result.get("edges", []), *semantic_result.get("edges", [])], + "hyperedges": [ + *ast_result.get("hyperedges", []), *semantic_result.get("hyperedges", []) + ], + "input_tokens": ast_result.get("input_tokens", 0) + semantic_result.get("input_tokens", 0), + "output_tokens": ast_result.get("output_tokens", 0) + semantic_result.get("output_tokens", 0), + } + build_data = ( + build_unclustered_extraction(result, root=build_root) + if no_cluster + else build_from_extraction(result, root=build_root) + ) + if had_existing_generation: + previous_topology_sources = previous_state.get("incremental", {}).get( + "topology_sources", [] + ) + expected_sources = { + str(source).replace("\\", "/") + for source in previous_topology_sources + if isinstance(source, str) + and (build_root / source).is_file() + } if isinstance(previous_topology_sources, list) else set() + candidate_sources = set(_topology_sources(build_data)) + missing_live_sources = expected_sources - candidate_sources + if missing_live_sources and not force: + sample = ", ".join(sorted(missing_live_sources)[:5]) print( - "[graphify watch] Rebuilt (no clustering): " - f"{len(candidate_graph_data.get('nodes', []))} nodes, " - f"{len(candidate_graph_data.get('links', []))} edges" + "[graphify] WARNING: extraction omitted live source file(s) " + f"present in the active generation ({sample}); pass --force " + "after confirming the extractor change.", + file=sys.stderr, ) - print(f"[graphify watch] graph.json updated in {out}") - return True + return False detection = { - "files": {"code": [str(f) for f in code_files], "document": [], "paper": [], "image": []}, - "total_files": len(code_files), + "files": {key: list(value) for key, value in files_by_type.items()}, + "total_files": detected.get("total_files", len(current_files)), "total_words": detected.get("total_words", 0), } - G = build_from_json(result) - candidate_topology = _topology_from_graph(G) - if existing_graph_data: - try: - same_topology = ( - json.dumps(_canonical_topology_for_compare(existing_graph_data), sort_keys=True, ensure_ascii=False) - == json.dumps(_canonical_topology_for_compare(candidate_topology), sort_keys=True, ensure_ascii=False) + def analyze_generation(graph, *, reuse_communities: bool = False): + if no_cluster: + communities = {} + cohesion = {} + labels = {} + elif reuse_communities: + communities = communities_from_state(previous_state) + labels = labels_from_state(previous_state) + cohesion = { + int(record["id"]): float(record["cohesion"]) + for record in previous_state.get("communities", []) + if isinstance(record, dict) + and isinstance(record.get("id"), int) + and isinstance(record.get("cohesion"), (int, float)) + } + else: + communities = cluster(graph) + cohesion = score_all(graph, communities) + labels = _community_labels(graph, communities) + if no_cluster: + gods = [] + surprises = [] + questions = [] + else: + gods = god_nodes(graph) + surprises = surprising_connections(graph, communities) + questions = suggest_questions(graph, communities, labels) + state = copy.deepcopy(previous_state) + state["build"] = { + "helix_python_version": HELIX_PYTHON_VERSION, + "node_count": graph.node_count, + "edge_count": graph.edge_count, + "directed": graph.directed, + "multigraph": graph.multigraph, + "semantic": bool(semantic_targets) + or bool(previous_state.get("semantic", {}).get("used")), + "source_root": str(source_root), + "source_commit": _git_head(watch_path), + } + state["communities"] = community_records( + communities, + labels=labels, + cohesion=cohesion, + naming_source="generated", + ) + state["analysis"] = { + "god_nodes": gods, + "surprises": surprises, + "suggested_questions": questions, + "confidence_counts": { + confidence: sum( + 1 + for edge in build_data.edges + if str(edge.attributes.get("confidence", "EXTRACTED")) + == confidence + ) + for confidence in ("EXTRACTED", "INFERRED", "AMBIGUOUS") + }, + "community_summaries": ( + [] + if no_cluster + else community_summaries(graph, communities, labels) + ), + "report_inputs": { + "detection": detection, + "tokens": { + "input": result.get("input_tokens", 0), + "output": result.get("output_tokens", 0), + }, + "source": str(watch_path), + }, + } + state["incremental"] = { + "files": { + path.relative_to(build_root).as_posix(): { + "content_hash": _content_hash(path), + "semantic_hash": "", + "extractor_state": "ast", + } + for path in current_files + if path.is_file() and path.is_relative_to(build_root) + }, + "extractor_state": { + "mode": "deep" if deep_mode else "semantic" if semantic_targets else "ast", + "backend": selected_backend if semantic_targets else None, + }, + "topology_sources": _topology_sources(build_data), + "extraction_cache": cache_state, + } + state["semantic"] = { + "used": bool(semantic_targets) + or bool(previous_state.get("semantic", {}).get("used")), + "backend": selected_backend + if semantic_targets + else previous_state.get("semantic", {}).get("backend"), + "input_tokens": result.get("input_tokens", 0), + "output_tokens": result.get("output_tokens", 0), + } + return communities, cohesion, labels, gods, surprises, questions, state + + with HelixEmbeddedStore( + store_path, retain_rollback=retain_rollback + ) as store: + topology_unchanged = ( + had_existing_generation and store.topology_matches(build_data) + ) + if no_cluster: + ( + communities, + cohesion, + labels, + gods, + surprises, + questions, + state, + ) = analyze_generation( + build_data, reuse_communities=topology_unchanged ) - except Exception: - same_topology = False - if same_topology: - try: - from graphify.detect import save_manifest - # Full-scan save: prune excluded-but-alive rows (#1908). - save_manifest( - detected["files"], kind="ast", root=project_root, - scan_corpus={f for _fl in detected["files"].values() for f in _fl}, + if topology_unchanged: + store.replace_state( + state, + previous_state=previous_state, ) - except Exception: - pass - flag = out / "needs_update" - if flag.exists(): - flag.unlink() - print("[graphify watch] No code-graph topology changes detected; outputs left untouched.") - return True - - communities = cluster(G) - previous_node_community = _node_community_map(existing_graph_data) - if previous_node_community: - communities = remap_communities_to_previous(communities, previous_node_community) - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - labels_file = out / ".graphify_labels.json" - try: - raw = json.loads(labels_file.read_text(encoding="utf-8")) if labels_file.exists() else {} - labels = {int(k): v for k, v in raw.items() if int(k) in communities} - except Exception: - raw = {} - labels = {} - missing = {cid: members for cid, members in communities.items() if cid not in labels} - if missing: - # Deterministic hub name (highest-degree member) beats a bare "Community N" - # placeholder for any community without a saved label. - from graphify.cluster import label_communities_by_hub - labels.update(label_communities_by_hub(G, missing)) - questions = suggest_questions(G, communities, labels) - from graphify.report import load_learning_for_report as _llfr - report = generate(G, communities, cohesion, labels, gods, surprises, detection, - {"input": 0, "output": 0}, report_root, suggested_questions=questions, - built_at_commit=commit, learning=_llfr(out / "graph.json")) - report_path = out / "GRAPH_REPORT.md" - labels_json = json.dumps({str(k): v for k, v in sorted(labels.items())}, ensure_ascii=False, indent=2) + "\n" - graph_tmp = out / ".graph.tmp.json" - json_written = to_json(G, communities, str(graph_tmp), force=True, built_at_commit=commit, community_labels=labels) - if not json_written: - return False - candidate_graph_data = json.loads(graph_tmp.read_text(encoding="utf-8")) - same_graph = False - same_report = False - if existing_graph.exists(): - try: - check_graph_file_size_cap(existing_graph) - existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) - same_graph = ( - json.dumps(_canonical_graph_for_compare(existing_payload), sort_keys=True, ensure_ascii=False) - == json.dumps(_canonical_graph_for_compare(candidate_graph_data), sort_keys=True, ensure_ascii=False) + print("[graphify watch] No code-graph topology changes detected.") + else: + store.save_generation(build_data, state) + graph_node_count = build_data.node_count + graph_edge_count = build_data.edge_count + elif topology_unchanged: + assert loaded is not None + graph = loaded.graph + ( + communities, + cohesion, + labels, + gods, + surprises, + questions, + state, + ) = analyze_generation(graph, reuse_communities=True) + store.replace_state( + state, + previous_state=previous_state, + snapshot=loaded, ) - except Exception: - same_graph = False - if report_path.exists(): - old_report = report_path.read_text(encoding="utf-8") - same_report = _report_for_compare(old_report) == _report_for_compare(report) - no_change = same_graph and same_report - if no_change: - graph_tmp.unlink(missing_ok=True) - print("[graphify watch] No code-graph changes detected; graph.json/GRAPH_REPORT.md left untouched.") - else: - if not _check_shrink( - force, existing_graph_data, candidate_graph_data, - tmp=graph_tmp, - had_explicit_deletions=bool(deleted_paths), - rebuilt_sources=rebuilt_sources, - ): - return False - from graphify.export import backup_if_protected as _backup - _backup(out) - graph_tmp.replace(existing_graph) - report_path.write_text(report, encoding="utf-8") - labels_file.write_text(labels_json, encoding="utf-8") + print("[graphify watch] No code-graph topology changes detected.") + else: + with store.staged_graph(build_data) as staged: + graph = staged.graph + ( + communities, + cohesion, + labels, + gods, + surprises, + questions, + state, + ) = analyze_generation(graph) + activated = store.activate_staged(staged, state) + graph = activated.graph - (out / ".graphify_root").write_text(str(watch_path), encoding="utf-8") + if no_cluster: + # Reopen the activated store through the ordinary public reader and + # verify its generation counts before reporting success. + with HelixEmbeddedStore(store_path, read_only=True) as reopened: + reopened_counts = reopened.verify_counts() + if ( + reopened_counts["nodes"] != graph_node_count + or reopened_counts["edges"] != graph_edge_count + ): + raise RuntimeError( + "activated Helix generation failed reopen verification" + ) + (out / ".graphify_root").write_text(_root_marker, encoding="utf-8") + (out / "needs_update").unlink(missing_ok=True) + print( + f"[graphify watch] Rebuilt (no clustering): {graph_node_count} nodes, " + f"{graph_edge_count} edges" + ) + print(f"[graphify watch] graph.helix updated in {out}") + return True + report = generate( + graph, communities, cohesion, labels, gods, surprises, detection, + { + "input": result.get("input_tokens", 0), + "output": result.get("output_tokens", 0), + }, str(watch_path), + suggested_questions=questions, built_at_commit=_git_head(watch_path), + learning=state.get("learning", {}), + ) + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") try: - from graphify.detect import save_manifest - # Full-scan save: prune excluded-but-alive rows (#1908). - save_manifest( - detected["files"], kind="ast", root=project_root, - scan_corpus={f for _fl in detected["files"].values() for f in _fl}, - ) - except Exception: - pass + to_html(graph, communities, str(out / "graph.html"), community_labels=labels) + except ValueError as exc: + print(f"[graphify watch] Skipped graph.html: {exc}") + (out / "graph.html").unlink(missing_ok=True) - # to_html raises ValueError for graphs > MAX_NODES_FOR_VIZ (5000). - # Wrap so core outputs (graph.json + GRAPH_REPORT.md) always land. - html_written = False - if not no_change: - try: - to_html(G, communities, str(out / "graph.html"), community_labels=labels or None) - html_written = True - except ValueError as viz_err: - print(f"[graphify watch] Skipped graph.html: {viz_err}") - stale = out / "graph.html" - if stale.exists(): - stale.unlink() - - # Regenerate callflow HTML if the user previously generated one — - # opt-in by existence so users who never ran callflow-html aren't affected. - callflow_files = list(out.glob("*-callflow.html")) - if callflow_files and not no_change: + for callflow in out.glob("*-callflow.html"): try: from graphify.callflow_html import write_callflow_html - for cf in callflow_files: - write_callflow_html( - graph=out / "graph.json", - report=out / "GRAPH_REPORT.md", - labels=out / ".graphify_labels.json", - output=cf, - verbose=False, - ) - except Exception as cf_err: - print(f"[graphify watch] callflow HTML update skipped: {cf_err}") - - # clear stale needs_update flag if present - flag = out / "needs_update" - if flag.exists(): - flag.unlink() - - if not no_change: - print(f"[graphify watch] Rebuilt: {G.number_of_nodes()} nodes, " - f"{G.number_of_edges()} edges, {len(communities)} communities") - products = "graph.json" + (", graph.html" if html_written else "") + " and GRAPH_REPORT.md" - if callflow_files: - products += f", {len(callflow_files)} callflow HTML" - print(f"[graphify watch] {products} updated in {out}") - return True + write_callflow_html( + graph=store_path, report=out / "GRAPH_REPORT.md", + output=callflow, + verbose=False, + ) + except Exception as exc: + print(f"[graphify watch] callflow HTML update skipped: {exc}") + + (out / ".graphify_root").write_text(_root_marker, encoding="utf-8") + (out / "needs_update").unlink(missing_ok=True) + print( + f"[graphify watch] Rebuilt: {graph.node_count} nodes, {graph.edge_count} edges, " + f"{len(communities)} communities" + ) + print(f"[graphify watch] graph.helix and GRAPH_REPORT.md updated in {out}") + return True except Exception as exc: + if raise_on_error: + raise print(f"[graphify watch] Rebuild failed: {exc}") return False def check_update(watch_path: Path) -> bool: - """Check for pending semantic update flag and notify the user if set. - - Cron-safe: always returns True so cron jobs do not alarm. - Non-code file changes (docs, papers, images) require LLM-backed - re-extraction via `/graphify --update` — this function only signals - that the update is needed. - """ flag = Path(watch_path) / _GRAPHIFY_OUT / "needs_update" if flag.exists(): print(f"[graphify check-update] Pending non-code changes in {watch_path}.") - print("[graphify check-update] Run `/graphify --update` to apply semantic re-extraction.") + print("[graphify check-update] Run `graphify --update` to apply semantic re-extraction.") return True def _notify_only(watch_path: Path) -> None: - """Write a flag file and print a notification (fallback for non-code-only corpora).""" flag = watch_path / _GRAPHIFY_OUT / "needs_update" flag.parent.mkdir(parents=True, exist_ok=True) flag.write_text("1", encoding="utf-8") - print(f"\n[graphify watch] New or changed files detected in {watch_path}") - print("[graphify watch] Non-code files changed - semantic re-extraction requires LLM.") - print("[graphify watch] Run `/graphify --update` in Claude Code to update the graph.") - print(f"[graphify watch] Flag written to {flag}") + print(f"\n[graphify watch] Non-code files changed in {watch_path}; run `graphify update .`.") def _has_non_code(changed_paths: list[Path]) -> bool: - return any(p.suffix.lower() not in _CODE_EXTENSIONS for p in changed_paths) - + return any(path.suffix.lower() not in _CODE_EXTENSIONS for path in changed_paths) -def watch(watch_path: Path, debounce: float = 3.0) -> None: - """ - Watch watch_path for new or modified files and auto-update the graph. - For code-only changes: re-runs AST extraction + rebuild immediately (no LLM). - For doc/paper/image changes: writes a needs_update flag and notifies the user - to run /graphify --update (LLM extraction required). - - debounce: seconds to wait after the last change before triggering (avoids - running on every keystroke when many files are saved at once). - """ +def watch( + watch_path: Path, + debounce: float = 3.0, + *, + retain_rollback: bool = False, +) -> None: try: + from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver - from watchdog.events import FileSystemEventHandler - except ImportError as e: - raise ImportError("watchdog not installed. Run: pip install watchdog") from e - - last_trigger: float = 0.0 - pending: bool = False - changed: set[Path] = set() - - # Load .graphifyignore patterns ONCE at startup so the handler does not - # re-parse the file on every filesystem event. Watchdog's handler runs on - # the observer thread and is invoked for every event the OS delivers - # (Time Machine writes, Docker/Colima VM I/O, Spotlight indexing, …) — - # without this short-circuit a busy volume can saturate a CPU core - # discarding events one extension at a time. (gh-928) - watch_root_for_ignore = watch_path.resolve() + except ImportError as exc: + raise ImportError("watchdog not installed. Run: pip install watchdog") from exc + + watch_path = Path(watch_path).resolve() + last_trigger = 0.0 + pending = False + changed: list[Path] = [] + ignore_root = watch_path.resolve() ignore_patterns = _load_graphifyignore( - watch_root_for_ignore, + ignore_root, gitignore=_read_build_gitignore(watch_path / _GRAPHIFY_OUT), ) @@ -1373,52 +1040,40 @@ def on_any_event(self, event): if event.is_directory: return path = Path(os.fsdecode(event.src_path)) - # Check .graphifyignore BEFORE the extension/dotfile/out filters so - # the cheapest short-circuit for users with broad ignore patterns - # (node_modules/, .venv/, build/, …) fires first. _is_ignored - # tolerates absolute paths outside watch_root via its internal - # relative_to guard, so a stray symlinked event won't raise. - if ignore_patterns and _is_ignored(path, watch_root_for_ignore, ignore_patterns): + if ignore_patterns and _is_ignored(path, ignore_root, ignore_patterns): return if path.suffix.lower() not in _WATCHED_EXTENSIONS: return try: - filter_parts = path.relative_to(watch_root_for_ignore).parts + filter_parts = path.relative_to(ignore_root).parts except ValueError: filter_parts = path.parts - if any(part.startswith(".") for part in filter_parts): - return - if _GRAPHIFY_OUT in filter_parts: + if any(part.startswith(".") for part in filter_parts) or _GRAPHIFY_OUT in filter_parts: return last_trigger = time.monotonic() pending = True - changed.add(path) + if path not in changed: + changed.append(path) - handler = Handler() - # Use polling observer on macOS — FSEvents can miss rapid saves in some editors observer = PollingObserver() if sys.platform == "darwin" else Observer() - observer.schedule(handler, str(watch_path), recursive=True) + observer.schedule(Handler(), str(watch_path), recursive=True) observer.start() - - print(f"[graphify watch] Watching {watch_path.resolve()} - press Ctrl+C to stop") - print(f"[graphify watch] Code changes rebuild graph automatically. " - f"Doc/image changes require /graphify --update.") - print(f"[graphify watch] Debounce: {debounce}s") - + print(f"[graphify watch] Watching {watch_path} - press Ctrl+C to stop") try: while True: time.sleep(0.5) - if pending and (time.monotonic() - last_trigger) >= debounce: + if pending and time.monotonic() - last_trigger >= debounce: pending = False batch = list(changed) changed.clear() - print(f"\n[graphify watch] {len(batch)} file(s) changed") - has_non_code = _has_non_code(batch) - has_code = any(p.suffix.lower() in _CODE_EXTENSIONS for p in batch) - if has_code: - _rebuild_code(watch_path) - if has_non_code: + if _has_non_code(batch): _notify_only(watch_path) + else: + _rebuild_code( + watch_path, + changed_paths=batch, + retain_rollback=retain_rollback, + ) except KeyboardInterrupt: print("\n[graphify watch] Stopped.") finally: @@ -1428,9 +1083,14 @@ def on_any_event(self, event): if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(description="Watch a folder and auto-update the graphify graph") - parser.add_argument("path", nargs="?", default=".", help="Folder to watch (default: .)") - parser.add_argument("--debounce", type=float, default=3.0, - help="Seconds to wait after last change before updating (default: 3)") + + parser = argparse.ArgumentParser(description="Watch a folder and update graph.helix") + parser.add_argument("path", nargs="?", default=".") + parser.add_argument("--debounce", type=float, default=3.0) + parser.add_argument("--retain-rollback", action="store_true") args = parser.parse_args() - watch(Path(args.path), debounce=args.debounce) + watch( + Path(args.path), + debounce=args.debounce, + retain_rollback=args.retain_rollback, + ) diff --git a/graphify/wiki.py b/graphify/wiki.py index cb9c6cf35..2aea8e229 100644 --- a/graphify/wiki.py +++ b/graphify/wiki.py @@ -3,10 +3,11 @@ from __future__ import annotations from collections import Counter from pathlib import Path +from typing import Any from urllib.parse import quote -import networkx as nx from graphify.build import edge_data +from graphify.helix.model import node_attributes def _safe_filename(name: str) -> str: @@ -48,7 +49,7 @@ def _md_link(label: str, resolver: dict[str, str]) -> str: return f"[{text}]({quote(f'{slug}.md')})" -def _cross_community_links(G: nx.Graph, nodes: list[str], own_cid: int, labels: dict[int, str], node_community: dict[str, int]) -> list[tuple[str, int]]: +def _cross_community_links(G: Any, nodes: list[str], own_cid: int, labels: dict[int, str], node_community: dict[str, int]) -> list[tuple[str, int]]: """Return (community_label, edge_count) pairs for cross-community connections, sorted descending.""" counts: dict[str, int] = Counter() for nid in nodes: @@ -60,7 +61,7 @@ def _cross_community_links(G: nx.Graph, nodes: list[str], own_cid: int, labels: def _community_article( - G: nx.Graph, + G: Any, cid: int, nodes: list[str], label: str, @@ -70,7 +71,7 @@ def _community_article( resolver: dict[str, str] | None = None, ) -> str: resolver = resolver or {} - top_nodes = sorted(nodes, key=lambda n: G.degree(n), reverse=True)[:25] + top_nodes = sorted(nodes, key=lambda n: G.degree(n).degree, reverse=True)[:25] cross = _cross_community_links(G, nodes, cid, labels, node_community or {}) # Edge confidence breakdown @@ -81,7 +82,7 @@ def _community_article( conf_counts[ed.get("confidence", "EXTRACTED")] += 1 total_edges = sum(conf_counts.values()) or 1 - sources = sorted({G.nodes[n].get("source_file") or "" for n in nodes} - {""}) + sources = sorted({node_attributes(G, n).get("source_file") or "" for n in nodes} - {""}) lines: list[str] = [] lines += [f"# {label}", ""] @@ -93,10 +94,10 @@ def _community_article( lines += ["## Key Concepts", ""] for nid in top_nodes: - d = G.nodes[nid] + d = node_attributes(G, nid) node_label = d.get("label", nid) src = d.get("source_file", "") - degree = G.degree(nid) + degree = G.degree(nid).degree src_str = f" — `{src}`" if src else "" lines.append(f"- **{node_label}** ({degree} connections){src_str}") remaining = len(nodes) - len(top_nodes) @@ -129,9 +130,9 @@ def _community_article( return "\n".join(lines) -def _god_node_article(G: nx.Graph, nid: str, labels: dict[int, str], node_community: dict[str, int] | None = None, resolver: dict[str, str] | None = None) -> str: +def _god_node_article(G: Any, nid: str, labels: dict[int, str], node_community: dict[str, int] | None = None, resolver: dict[str, str] | None = None) -> str: resolver = resolver or {} - d = G.nodes[nid] + d = node_attributes(G, nid) node_label = d.get("label", nid) src = d.get("source_file", "") cid = (node_community or {}).get(nid) @@ -139,15 +140,15 @@ def _god_node_article(G: nx.Graph, nid: str, labels: dict[int, str], node_commun lines: list[str] = [] lines += [f"# {node_label}", ""] - lines += [f"> God node · {G.degree(nid)} connections · `{src}`", ""] + lines += [f"> God node · {G.degree(nid).degree} connections · `{src}`", ""] if community_name: lines += [f"**Community:** {_md_link(community_name, resolver)}", ""] # Group neighbors by relation type by_relation: dict[str, list[str]] = {} - for neighbor in sorted(G.neighbors(nid), key=lambda n: G.degree(n), reverse=True): - nd = G.nodes[neighbor] + for neighbor in sorted(G.neighbors(nid), key=lambda n: G.degree(n).degree, reverse=True): + nd = node_attributes(G, neighbor) ed = edge_data(G, nid, neighbor) rel = ed.get("relation", "related") neighbor_label = nd.get("label", neighbor) @@ -203,13 +204,13 @@ def _index_md( lines += [ "---", "", - "*Generated by [graphify](https://github.com/safishamsi/graphify)*", + "*Generated by [graphify](https://github.com/Graphify-Labs/graphify)*", ] return "\n".join(lines) def to_wiki( - G: nx.Graph, + G: Any, communities: dict[int, list[str]], output_dir: str | Path, community_labels: dict[int, str] | None = None, @@ -234,12 +235,9 @@ def to_wiki( "Run `graphify extract .` or `graphify cluster-only .` first." ) - # Filter stale node IDs that exist in communities but not in G. - # Analysis JSON can drift from the graph after dedup / re-extract / update. - # NetworkX 3.x returns DegreeView({}) for missing nodes instead of raising, - # which crashes sorted() with TypeError; G.neighbors()/G.nodes[] also raise. + # Filter stale community IDs that no longer exist in the active generation. import sys as _sys - _g_nodes = set(G.nodes) + _g_nodes = {node.id for node in G.nodes()} _orig_total = sum(len(ns) for ns in communities.values()) communities = {cid: [n for n in nodes if n in _g_nodes] for cid, nodes in communities.items()} communities = {cid: nodes for cid, nodes in communities.items() if nodes} @@ -254,7 +252,7 @@ def to_wiki( if not communities: raise ValueError( "all community node IDs are stale — none exist in the graph. " - "Re-run `graphify extract .` to regenerate .graphify_analysis.json." + "Re-run `graphify extract .` to regenerate native analysis state." ) # Clear stale .md files from previous runs to prevent orphan accumulation. @@ -269,6 +267,9 @@ def to_wiki( labels = community_labels or {cid: f"Community {cid}" for cid in communities} cohesion = cohesion or {} god_nodes_data = god_nodes_data or [] + membership = { + node: cid for cid, members in communities.items() for node in members + } # Build node->community lookup once; node attrs never carry community (it lives in # the communities dict), so _cross_community_links and _god_node_article need this. @@ -311,7 +312,7 @@ def _unique_slug(base: str) -> str: god_articles: list[tuple[str, str]] = [] # (node_id, slug) for node_data in god_nodes_data: nid = node_data.get("id") - if nid and nid in G: + if nid and G.contains_node(nid): slug = _unique_slug(_safe_filename(node_data['label'])) god_articles.append((nid, slug)) resolver.setdefault(node_data['label'], slug) @@ -330,7 +331,7 @@ def _unique_slug(base: str) -> str: # Index (out / "index.md").write_text( - _index_md(communities, labels, god_nodes_data, G.number_of_nodes(), G.number_of_edges(), resolver), + _index_md(communities, labels, god_nodes_data, G.node_count, G.edge_count, resolver), encoding="utf-8", ) diff --git a/pyproject.toml b/pyproject.toml index f64ae84f0..1cb581c20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68"] +requires = ["setuptools>=83.0.0"] build-backend = "setuptools.build_meta" [project] @@ -11,7 +11,9 @@ license = { file = "LICENSE" } keywords = ["claude", "claude-code", "codex", "opencode", "kilo", "cursor", "gemini", "aider", "kiro", "pi", "devin", "knowledge-graph", "rag", "graphrag", "obsidian", "community-detection", "tree-sitter", "leiden", "llm"] requires-python = ">=3.10" dependencies = [ - "networkx>=3.4", + "helix-db==0.2.0b3", + "helix-db-embedded==0.2.0b3", + "defusedxml>=0.7.1", "numpy>=1.21", "rapidfuzz>=3.0", "tree-sitter>=0.23.0,<0.26", @@ -51,13 +53,12 @@ Issues = "https://github.com/Graphify-Labs/graphify/issues" # starlette is pulled in transitively by mcp, but graphify/serve.py imports it # directly for the HTTP transport, so declare it here and floor it above the # CVE-2026-48818 / CVE-2026-54283 fixes (both resolved by 1.3.1) (#1391, #1396). -mcp = ["mcp", "starlette>=1.3.1"] +mcp = ["mcp>=1.28.1", "starlette>=1.3.1"] neo4j = ["neo4j"] falkordb = ["falkordb"] pdf = ["pypdf>=6.12.0", "markdownify"] watch = ["watchdog"] svg = ["matplotlib", "numpy>=2.0; python_version >= '3.13'"] -leiden = ["graspologic; python_version < '3.13'"] office = ["python-docx", "openpyxl"] google = ["openpyxl"] postgres = ["psycopg[binary]"] @@ -80,7 +81,7 @@ pascal = ["tree-sitter-pascal"] # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +all = ["mcp>=1.28.1", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] [project.scripts] graphify = "graphify.__main__:main" @@ -99,7 +100,7 @@ dev = [ "pytest>=9.0.3", "pytest-cov>=7.1.0", "ruff>=0.15.13", - "setuptools>=82.0.1", + "setuptools>=83.0.0", "wheel>=0.47.0", "tomli>=2.0 ; python_version < '3.11'", "tree-sitter-hcl>=1.2.0", @@ -111,7 +112,7 @@ dev = [ package = true [tool.setuptools] -packages = ["graphify", "graphify.extractors", "graphify.exporters"] +packages = ["graphify", "graphify.extractors", "graphify.exporters", "graphify.helix"] include-package-data = false [tool.setuptools.package-data] @@ -145,3 +146,8 @@ select = ["E9", "F63", "F7", "F82"] include = ["graphify", "tests"] pythonVersion = "3.10" typeCheckingMode = "basic" + +[tool.graphify.helix] +index = "https://pypi.org/project/helix-db/0.2.0b3/" +package = "helix-db==0.2.0b3" +embedded_package = "helix-db-embedded==0.2.0b3" diff --git a/requirements-runtime.txt b/requirements-runtime.txt new file mode 100644 index 000000000..84e229bc6 --- /dev/null +++ b/requirements-runtime.txt @@ -0,0 +1,33 @@ +# Production dependencies. Keep synchronized with pyproject.toml. +# Public Helix SDK and its matching embedded runtime. +helix-db==0.2.0b3 +helix-db-embedded==0.2.0b3 +defusedxml>=0.7.1 +numpy>=1.21 +rapidfuzz>=3.0 +tree-sitter>=0.23.0,<0.26 +tree-sitter-python>=0.23,<0.26 +tree-sitter-javascript>=0.23,<0.26 +tree-sitter-typescript>=0.23,<0.25 +tree-sitter-go>=0.23,<0.26 +tree-sitter-rust>=0.23,<0.25 +tree-sitter-java>=0.23,<0.25 +tree-sitter-groovy>=0.1,<0.3 +tree-sitter-c>=0.23,<0.25 +tree-sitter-cpp>=0.23,<0.25 +tree-sitter-ruby>=0.23,<0.25 +tree-sitter-c-sharp>=0.23,<0.25 +tree-sitter-kotlin>=1.0,<2.0 +tree-sitter-scala>=0.23,<0.27 +tree-sitter-php>=0.23,<0.25 +tree-sitter-swift>=0.7,<0.9 +tree-sitter-lua>=0.2,<0.6 +tree-sitter-zig>=1.0,<2.0 +tree-sitter-powershell>=0.26,<0.28 +tree-sitter-elixir>=0.3,<0.5 +tree-sitter-objc>=3.0,<4.0 +tree-sitter-julia>=0.23,<0.25 +tree-sitter-verilog>=1.0,<2.0 +tree-sitter-fortran>=0.6,<0.8 +tree-sitter-bash>=0.23,<0.27 +tree-sitter-json>=0.23,<0.26 diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb..420c66ee8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Graphify test package.""" diff --git a/tests/bench_query_scoring.py b/tests/bench_query_scoring.py deleted file mode 100644 index 68e6ce4be..000000000 --- a/tests/bench_query_scoring.py +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env python3 -"""Non-CI microbenchmark: single-pass scoring vs legacy per-term rescoring. - -Verifies the single-pass refactor eliminates the T+1 graph-scoring passes a -T-term query used to run, without changing the result. Reports median/min -latency for: - - * legacy — one combined `_score_nodes(G, terms)` call PLUS one - `_score_nodes(G, [token])` call per distinct query token - (the old `_pick_seeds(terms=...)` per-term guarantee loop). - * optimized — one `_score_query(G, terms, collect_per_term_seeds=True)` - call, with the per-token best computed inline in the same - traversal. `_pick_seeds` then consumes `best_seed_by_term` - directly — no rescoring. - -Equality is asserted (ranked, best_seed_by_term, seeds) before timing. The -optimized path's traversal count is invariant in the number of query terms. - -Run it manually; do NOT wire this into CI (wall-clock assertions are flaky): - - uv run python tests/bench_query_scoring.py \\ - --nodes 100000 --term-counts 3,10 --repeats 5 - - uv run python tests/bench_query_scoring.py \\ - --graph graphify-out/graph.json \\ - --query "what calls extract" --query "symbol resolution" \\ - --repeats 10 -""" -from __future__ import annotations - -import argparse -import json -import random -import statistics -import sys -import time -from pathlib import Path - -import networkx as nx - -from graphify.serve import ( - _get_trigram_index, - _pick_seeds, - _query_terms, - _score_nodes, - _score_query, - _search_tokens, -) - - -SYLLABLES = [ - "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", - "build", "report", "extract", "router", "config", "service", - "handler", "token", "auth", "rate", "limit", "widget", "model", -] - -QUERIES_BY_TERM_COUNT: dict[int, list[str]] = { - 1: ["foo"], - 2: ["foo", "bar"], - 3: ["router", "service", "handler"], - 5: ["get", "user", "run", "name", "path"], - 10: ["extract", "build", "report", "router", "config", - "service", "token", "rate", "limit", "widget"], -} - - -def _build_random_graph(n: int, *, seed: int) -> nx.DiGraph: - """Reproducible broad-match DiGraph: short constructed labels + edge noise. - - Labels draw from a small syllable pool so tokens collide across nodes, - forcing the trigram prefilter to be selective and exercising score ties - on common tokens. Edge noise provides degree variance so the legacy - tie-break (`max(tied, key=degree)`) is actually exercised against the - new `(-singleton, -degree, label_len, nid)` key tuple.""" - rng = random.Random(seed) - G: nx.DiGraph = nx.DiGraph() - for i in range(n): - label = "_".join(rng.sample(SYLLABLES, rng.randint(1, 3))) - G.add_node(f"n{i}", label=label, source_file=f"src/{label[:8]}.py") - for _ in range(n * 2): - a, b = rng.randrange(n), rng.randrange(n) - if a != b: - G.add_edge(f"n{a}", f"n{b}", relation="calls", confidence="EXTRACTED") - return G - - -def _load_real_graph(path: str) -> nx.Graph: - """Light wrapper that just builds a NetworkX graph from a real - `graphify-out/graph.json`, skipping the size cap and work-memory overlay - that `serve._load_graph` enforces (the bench is read-only).""" - data = json.loads(Path(path).read_text(encoding="utf-8")) - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - data = {**data, "directed": True} - try: - return nx.readwrite.json_graph.node_link_graph(data, edges="links") - except TypeError: - return nx.readwrite.json_graph.node_link_graph(data) - - -def _legacy_score_and_pick(G, terms): - """Recreates the pre-refactor flow: combined scoring plus one - `_score_nodes([token])` call per distinct query token, with the legacy - tie-break (`max(tied, key=degree)` over a (-score, label_len, nid)-sorted - list) to derive `best_seed_by_term`. Returns the same triple as the - optimized path so they can be compared for equality.""" - ranked = _score_nodes(G, terms) - norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) - best_seed_by_term: dict[str, str] = {} - for term in norm_terms: - term_scored = _score_nodes(G, [term]) - if not term_scored: - continue - best_score = term_scored[0][0] - tied = [nid for s, nid in term_scored if s == best_score] - best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] - best_seed_by_term[term] = best_nid - seeds = _pick_seeds(ranked, G=G, best_seed_by_term=best_seed_by_term) - return ranked, best_seed_by_term, seeds - - -def _optimized_score_and_pick(G, terms): - qs = _score_query(G, terms, collect_per_term_seeds=True) - seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) - return qs.ranked, qs.best_seed_by_term, seeds - - -def _warm_caches(G, terms): - """Pre-populate the trigram index and IDF cache so the first timed - iteration doesn't pay the amortized build cost on either path. Both - legacy and optimized share these caches via the graph object, so warming - once is fair to both.""" - _get_trigram_index(G) - # Touch the idf cache for the combined terms and every per-token singleton, - # matching exactly the calls the legacy path will make. - _score_nodes(G, terms) - for term in {tok for t in terms for tok in _search_tokens(t)}: - _score_nodes(G, [term]) - - -def _bench(fn, *, repeats: int) -> list[float]: - # One uncounted warm-up — `_warm_caches` already populated caches, but - # this also amortizes any per-call code-path setup unique to `fn`. - fn() - times: list[float] = [] - for _ in range(repeats): - t0 = time.perf_counter() - fn() - times.append(time.perf_counter() - t0) - return times - - -def _verify_equality(G, terms) -> tuple[int, int]: - leg_rank, leg_best, leg_seeds = _legacy_score_and_pick(G, terms) - opt_rank, opt_best, opt_seeds = _optimized_score_and_pick(G, terms) - assert leg_rank == opt_rank, ( - f"ranked diverged for terms={terms}: legacy[:5]={leg_rank[:5]} opt[:5]={opt_rank[:5]}" - ) - assert leg_best == opt_best, ( - f"best_seed_by_term diverged for terms={terms}: legacy={leg_best} opt={opt_best}" - ) - assert leg_seeds == opt_seeds, ( - f"seeds diverged for terms={terms}: legacy={leg_seeds} opt={opt_seeds}" - ) - return len(leg_rank), len(leg_seeds) - - -def _row(label: str, n_nodes: int, n_terms: int, times: list[float], - traversal_count: int, n_ranked: int, n_seeds: int) -> str: - med = statistics.median(times) * 1000 - mn = min(times) * 1000 - return (f"{label:<10} | n={n_nodes:<7} | terms={n_terms:<3} | " - f"median={med:7.2f}ms | min={mn:7.2f}ms | " - f"passes={traversal_count:<3} | ranked={n_ranked:<6} seeds={n_seeds}") - - -def _legacy_traversal_count(terms) -> int: - # 1 combined pass + one per-token singleton pass. - return 1 + len({tok for t in terms for tok in _search_tokens(t)}) - - -def _run_scenario(G, terms, *, repeats: int) -> tuple[float, float]: - _warm_caches(G, terms) - n_ranked, n_seeds = _verify_equality(G, terms) - - legacy_times = _bench(lambda: _legacy_score_and_pick(G, terms), repeats=repeats) - opt_times = _bench(lambda: _optimized_score_and_pick(G, terms), repeats=repeats) - - n_nodes = G.number_of_nodes() - n_terms = len(set(tok for t in terms for tok in _search_tokens(t))) - print(_row("legacy", n_nodes, n_terms, legacy_times, - _legacy_traversal_count(terms), n_ranked, n_seeds)) - print(_row("optimized", n_nodes, n_terms, opt_times, - 1, n_ranked, n_seeds)) - - med_legacy = statistics.median(legacy_times) - med_opt = statistics.median(opt_times) - speedup = med_legacy / med_opt if med_opt > 0 else float("inf") - print(f"speedup | median: {speedup:.2f}x | " - f"min: {min(legacy_times) / min(opt_times):.2f}x") - return med_legacy, med_opt - - -def _resolve_scenarios(args) -> list[list[str]]: - if args.graph: - # Real-graph mode: each --query is a natural-language sentence, tokenized - # using the same helper the production path uses. - sentences = args.query or ["what calls extract"] - scenarios = [_query_terms(s) for s in sentences] - # Dedupe identical token sets (multiple --query args may tokenize the same). - seen: list[list[str]] = [] - for q in scenarios: - if q not in seen: - seen.append(q) - return seen - term_counts = [int(s.strip()) for s in args.term_counts.split(",") if s.strip()] - scenarios: list[list[str]] = [] - for tc in term_counts: - if tc in QUERIES_BY_TERM_COUNT: - scenarios.append(QUERIES_BY_TERM_COUNT[tc]) - elif tc > 0: - scenarios.append([SYLLABLES[i % len(SYLLABLES)] for i in range(tc)]) - return scenarios - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser( - description="Microbenchmark single-pass query scoring vs legacy per-term rescoring.", - ) - p.add_argument("--nodes", type=int, default=100_000, - help="node count for the synthetic benchmark graph (default: 100000)") - p.add_argument("--seed", type=int, default=20260714, help="RNG seed for the synthetic graph") - p.add_argument("--term-counts", default="3,10", - help="comma-separated list of term counts to benchmark (synthetic mode)") - p.add_argument("--repeats", type=int, default=5, - help="timed iterations per scenario (after one warm-up)") - p.add_argument("--graph", default=None, - help="optional path to a real graphify-out/graph.json; overrides --nodes") - p.add_argument("--query", action="append", default=None, - help="natural-language query sentence (real-graph mode; repeat for multiple)") - args = p.parse_args(argv) - - if args.graph: - print(f"loading real graph: {args.graph} ...", file=sys.stderr) - t0 = time.perf_counter() - G = _load_real_graph(args.graph) - print(f" loaded in {time.perf_counter() - t0:.2f}s", file=sys.stderr) - else: - print(f"building synthetic graph: n={args.nodes} seed={args.seed} ...", - file=sys.stderr) - t0 = time.perf_counter() - G = _build_random_graph(args.nodes, seed=args.seed) - print(f" built in {time.perf_counter() - t0:.2f}s", file=sys.stderr) - - print() - print(f"graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") - scenarios = _resolve_scenarios(args) - print(f"scenarios: {len(scenarios)} | repeats per scenario: {args.repeats}") - print("-" * 110) - - summaries = [] - for terms in scenarios: - print() - med_legacy, med_opt = _run_scenario(G, terms, repeats=args.repeats) - summaries.append((terms, med_legacy, med_opt)) - - print() - print("-" * 110) - print("summary:") - for terms, med_legacy, med_opt in summaries: - speedup = med_legacy / med_opt if med_opt > 0 else float("inf") - print(f" terms={len(set(tok for t in terms for tok in _search_tokens(t))):>3} | " - f"median legacy={med_legacy*1000:>8.2f}ms | " - f"median optimized={med_opt*1000:>8.2f}ms | " - f"speedup={speedup:>5.2f}x") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/native_helpers.py b/tests/native_helpers.py new file mode 100644 index 000000000..6dee9aae0 --- /dev/null +++ b/tests/native_helpers.py @@ -0,0 +1,74 @@ +"""Real embedded-Helix fixtures used by Graphify integration tests.""" + +from __future__ import annotations + +from pathlib import Path +import tempfile +from typing import Any + +from graphify.helix.model import GraphBuildData +from graphify.helix.persistence import HelixEmbeddedStore +from graphify.helix.state import new_state + + +def make_loaded( + root: Path | None = None, + *, + nodes: list[dict[str, Any]] | None = None, + edges: list[dict[str, Any]] | None = None, + kind: str = "graph", + graph: dict[str, Any] | None = None, + state: dict[str, Any] | None = None, +): + directed = kind in {"digraph", "multidigraph"} + multigraph = kind in {"multigraph", "multidigraph"} + root = Path(tempfile.mkdtemp(prefix="graphify-native-test-")) if root is None else root + payload = { + "directed": directed, + "multigraph": multigraph, + "graph": graph or {}, + "nodes": nodes or [], + "links": edges or [], + } + build = GraphBuildData.from_node_link(payload) + store_path = root / "graph.helix" + with HelixEmbeddedStore(store_path) as store: + store.save_generation(build, state or new_state()) + with HelixEmbeddedStore(store_path, read_only=True) as store: + return store.load() + + +def triangle(root: Path): + return make_loaded( + root, + nodes=[ + {"id": "a", "label": "A", "source_file": "a.py", "file_type": "code"}, + {"id": "b", "label": "B", "source_file": "b.py", "file_type": "code"}, + {"id": "c", "label": "C", "source_file": "c.py", "file_type": "code"}, + ], + edges=[ + {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED", "weight": 1.0}, + {"source": "b", "target": "c", "relation": "imports", "confidence": "INFERRED", "weight": 0.8}, + {"source": "c", "target": "a", "relation": "references", "confidence": "AMBIGUOUS", "weight": 0.5}, + ], + ) + + +def graph_from_payload( + nodes: list[dict[str, Any]], + edges: list[dict[str, Any]] | None = None, + *, + kind: str = "graph", +): + """Return a real immutable embedded-Helix snapshot for unit tests.""" + return make_loaded(nodes=nodes, edges=edges or [], kind=kind).graph + + +def graph_from_build(build: GraphBuildData): + """Persist transient build data and return a real native snapshot.""" + root = Path(tempfile.mkdtemp(prefix="graphify-native-build-test-")) + store_path = root / "graph.helix" + with HelixEmbeddedStore(store_path) as store: + store.save_generation(build, new_state()) + with HelixEmbeddedStore(store_path, read_only=True) as store: + return store.load().graph diff --git a/tests/test_affected_cli.py b/tests/test_affected_cli.py index a50e4bc27..14efe891f 100644 --- a/tests/test_affected_cli.py +++ b/tests/test_affected_cli.py @@ -1,270 +1,58 @@ -from __future__ import annotations - -import json - -import networkx as nx -from networkx.readwrite import json_graph - -import graphify.__main__ as mainmod - - -def _write_graph(tmp_path): - graph = nx.DiGraph() - graph.add_node("target", label="Foo", source_file="pkg/foo.py", source_location="L1") - graph.add_node("caller", label="X()", source_file="app.py", source_location="L4") - graph.add_node("barrel", label="__init__.py", source_file="pkg/__init__.py", source_location=None) - graph.add_node("consumer", label="app.py", source_file="app.py", source_location=None) - graph.add_edge("caller", "target", relation="calls", context="call", confidence="EXTRACTED") - graph.add_edge("barrel", "target", relation="re_exports", context="export", confidence="EXTRACTED") - graph.add_edge("consumer", "target", relation="imports", context="import", confidence="EXTRACTED") - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links")), encoding="utf-8") - return graph_path - - -def test_affected_cli_reverse_traverses_impact_edges(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr( - mainmod.sys, - "argv", - ["graphify", "affected", "Foo", "--graph", str(graph_path)], - ) - - mainmod.main() - - out = capsys.readouterr().out - assert "Affected nodes for Foo" in out - assert "X()" in out - assert "calls" in out - assert "__init__.py" in out - assert "re_exports" in out - assert "app.py" in out - assert "imports" in out - - -def test_affected_cli_relation_filter_limits_reverse_traversal(monkeypatch, tmp_path, capsys): - graph_path = _write_graph(tmp_path) +from graphify import __main__ as mainmod +from graphify.affected import affected_nodes, resolve_seed +from tests.native_helpers import make_loaded + + +def _loaded(tmp_path): + return make_loaded( + tmp_path, + kind="digraph", + nodes=[ + {"id": "foo", "label": "Foo()", "source_file": "foo.py", "file_type": "code"}, + {"id": "bar", "label": "Bar", "source_file": "bar.py", "file_type": "code"}, + {"id": "baz", "label": "Baz", "source_file": "baz.py", "file_type": "code"}, + ], + edges=[ + {"source": "bar", "target": "foo", "relation": "calls"}, + {"source": "baz", "target": "bar", "relation": "imports"}, + ], + ) + + +def test_native_reverse_traversal_and_relation_filter(tmp_path): + loaded = _loaded(tmp_path) + graph = loaded.graph + assert resolve_seed(graph, "Foo", node_query=loaded.query) == "foo" + all_hits = affected_nodes(graph, "foo", relations={"calls", "imports"}, depth=2) + assert {row.node_id for row in all_hits} == {"bar", "baz"} + calls = affected_nodes(graph, "foo", relations={"calls"}, depth=2) + assert {row.node_id for row in calls} == {"bar"} + + +def test_cli_reads_helix_store(tmp_path, monkeypatch, capsys): + loaded = _loaded(tmp_path) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, "argv", - ["graphify", "affected", "Foo", "--relation", "calls", "--graph", str(graph_path)], + ["graphify", "affected", "Foo", "--store", str(loaded.store_path)], ) - mainmod.main() - - out = capsys.readouterr().out - assert "Relations: calls" in out - assert "X()" in out - assert "__init__.py" not in out - - -def test_affected_cli_forces_directed_on_undirected_graph(monkeypatch, tmp_path, capsys): - """A graph persisted with directed=false must still recover caller->callee - direction (#1174): affected on the callee returns the caller, not the callee - or nothing. Without forcing directed=True, node_link_graph builds an - undirected Graph, predecessors() collapses, and the reverse traversal breaks. - """ - graph = nx.DiGraph() - graph.add_node("A", label="caller_fn", source_file="a.py", source_location="L1") - graph.add_node("B", label="callee_fn", source_file="b.py", source_location="L2") - graph.add_edge("A", "B", relation="calls", context="call", confidence="EXTRACTED") - - data = json_graph.node_link_data(graph, edges="links") - # Persist as undirected on disk to reproduce the bug condition. - data["directed"] = False - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(data), encoding="utf-8") - - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr( - mainmod.sys, - "argv", - ["graphify", "affected", "B", "--relation", "calls", "--graph", str(graph_path)], - ) - - mainmod.main() - - out = capsys.readouterr().out - # A (the caller) is affected by a change to B (the callee). - assert "caller_fn" in out - assert "calls" in out - # B is the query node, not an affected node, and the result is not empty. - assert "No affected nodes found." not in out - - -def test_affected_cli_loads_edges_keyed_graph(monkeypatch, tmp_path, capsys): - """graphify's `extract` writes graph.json with an "edges" key (not networkx's - default "links"). affected.load_graph must handle it; before the edges/links - normalization it raised an uncaught KeyError: 'links' (same class as #1198).""" - graph = nx.DiGraph() - graph.add_node("target", label="Foo", source_file="pkg/foo.py", source_location="L1") - graph.add_node("caller", label="X()", source_file="app.py", source_location="L4") - graph.add_edge("caller", "target", relation="calls", context="call", confidence="EXTRACTED") - - # Emulate graphify extract output: top-level "edges" key instead of "links". - data = json_graph.node_link_data(graph, edges="links") - data["edges"] = data.pop("links") - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(data), encoding="utf-8") - - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr( - mainmod.sys, - "argv", - ["graphify", "affected", "Foo", "--graph", str(graph_path)], - ) - - mainmod.main() - - out = capsys.readouterr().out - assert "Affected nodes for Foo" in out - assert "X()" in out - assert "calls" in out + output = capsys.readouterr().out + assert "Bar" in output and "Baz" in output -def test_resolve_seed_bare_name_matches_callable_label(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="classifyProperty()", source_file="pkg/entity.py") - graph.add_node("b", label="classifyPropertySafe()", source_file="app/context.py") - - assert resolve_seed(graph, "classifyProperty") == "a" - assert resolve_seed(graph, "classifyPropertySafe") == "b" - - -def test_resolve_seed_decorated_query_matches_bare_label(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="Foo", source_file="pkg/foo.py") - graph.add_node("b", label="FooBar", source_file="pkg/foobar.py") - - assert resolve_seed(graph, "Foo()") == "a" - - -def test_resolve_seed_matches_unicode_normalized_label(): - import unicodedata - - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="Auditoría", source_file="pkg/auditoria.py") - - assert resolve_seed(graph, unicodedata.normalize("NFD", "Auditoría")) == "a" - - -def test_resolve_seed_preserves_distinct_accents(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="resume", source_file="pkg/resume.py") - graph.add_node("b", label="résumé", source_file="pkg/resume_accented.py") - - assert resolve_seed(graph, "resume") == "a" - - -def test_resolve_seed_bare_name_tie_still_returns_none(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - graph.add_node("a", label="dup()", source_file="pkg/one.py") - graph.add_node("b", label="dup()", source_file="pkg/two.py") - - assert resolve_seed(graph, "dup") is None - - -def test_resolve_seed_source_file_path_prefers_file_level_node(): - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - source_file = "app/api/example/route.ts" - graph.add_node( - "example_route_get", - label="GET()", - source_file=source_file, - source_location="L42", - ) - graph.add_node( - "example_route", - label="route.ts", - source_file=source_file, - source_location="L1", - ) - - assert resolve_seed(graph, source_file) == "example_route" - - -def test_resolve_seed_source_file_trailing_slash_parity(): - """A trailing path separator must not change the match (parity with explain's - _find_node, which tokenizes the path and drops the slash).""" - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - source_file = "app/api/example/route.ts" - graph.add_node("get", label="GET()", source_file=source_file, source_location="L42") - graph.add_node("file", label="route.ts", source_file=source_file, source_location="L1") - - assert resolve_seed(graph, source_file + "/") == "file" - - -def test_resolve_seed_source_file_ambiguous_no_file_node_returns_none(): - """Several nodes share a source_file but none is the L1 file node and none's - basename matches the path — must not guess; return None.""" - from graphify.affected import resolve_seed - - graph = nx.DiGraph() - source_file = "pkg/handlers.py" - graph.add_node("a", label="handle_a()", source_file=source_file, source_location="L10") - graph.add_node("b", label="handle_b()", source_file=source_file, source_location="L20") - - assert resolve_seed(graph, source_file) is None - - -def test_affected_cli_source_file_path_uses_file_level_node(monkeypatch, tmp_path, capsys): - graph = nx.DiGraph() - source_file = "app/api/example/route.ts" - graph.add_node( - "example_route_get", - label="GET()", - source_file=source_file, - source_location="L42", - ) - graph.add_node( - "example_route", - label="route.ts", - source_file=source_file, - source_location="L1", - ) - graph.add_node( - "consumer", - label="consumer.ts", - source_file="app/consumer.ts", - source_location="L1", - ) - graph.add_edge( - "consumer", - "example_route", - relation="imports_from", - context="import", - confidence="EXTRACTED", - ) - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps(json_graph.node_link_data(graph, edges="links")), encoding="utf-8") - +def test_cli_rejects_legacy_json_path(tmp_path, monkeypatch, capsys): + legacy = tmp_path / "legacy.json" + legacy.write_text("{}") monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, "argv", - ["graphify", "affected", source_file, "--graph", str(graph_path)], + ["graphify", "affected", "Foo", "--store", str(legacy)], ) - - mainmod.main() - - out = capsys.readouterr().out - assert "Affected nodes for route.ts" in out - assert "consumer.ts" in out - assert "imports_from" in out - assert "No unique node matched" not in out + try: + mainmod.main() + except SystemExit as exc: + assert exc.code == 1 + assert "obsolete" in capsys.readouterr().err diff --git a/tests/test_affected_member_seed.py b/tests/test_affected_member_seed.py index d1f6b3ae7..2b5f07d23 100644 --- a/tests/test_affected_member_seed.py +++ b/tests/test_affected_member_seed.py @@ -1,61 +1,20 @@ -"""#1669 — affected must reach callers that bind to the class's method -nodes (post-#1634 method-granularity resolution), by seeding the reverse walk -with the root's member nodes (one method/contains hop). method/contains stay out -of the general relation-filtered walk, so no forward noise is added elsewhere. -""" -from __future__ import annotations - -import networkx as nx - from graphify.affected import affected_nodes - - -def _g(): - g = nx.DiGraph() - for nid, label in [ - ("proc", "Processor"), ("proc_call", ".call()"), - ("runner", "Runner"), ("runner_run", ".run()"), - ]: - g.add_node(nid, label=label) - g.add_edge("proc", "proc_call", relation="method") # class owns method - g.add_edge("runner", "runner_run", relation="method") - g.add_edge("runner_run", "proc_call", relation="calls") # caller binds to method node (#1634) - return g - - -def test_class_affected_reaches_method_bound_caller(): - g = _g() - hits = {h.node_id for h in affected_nodes(g, "proc", depth=2)} - assert "runner_run" in hits, "caller of Processor.call must be reachable from Processor" - - -def test_member_method_node_not_reported_as_hit(): - g = _g() - hits = {h.node_id for h in affected_nodes(g, "proc", depth=2)} - # the class's own method node is a seed, not an affected node - assert "proc_call" not in hits - - -def test_method_contains_still_excluded_from_general_walk(): - # A node two method-hops away (method of a DIFFERENT class discovered during - # the walk) must NOT be pulled in: only the root's own members are seeded. - g = nx.DiGraph() - for nid, label in [("a", "A"), ("a_m", ".m()"), ("b", "B"), ("b_m", ".n()")]: - g.add_node(nid, label=label) - g.add_edge("a", "a_m", relation="method") - g.add_edge("a_m", "b", relation="calls") # A.m calls class B - g.add_edge("b", "b_m", relation="method") # B's own method - hits = {h.node_id for h in affected_nodes(g, "a", depth=3)} - # We seeded A's members and walk reverse; B and B's method are downstream of A - # (A.m -> B), not reverse-callers of A, so they must not appear. - assert hits == set() or "b_m" not in hits - - -def test_class_level_caller_still_works(): - # A caller bound to the class node itself (not a method) is unaffected. - g = nx.DiGraph() - g.add_node("svc", label="Svc") - g.add_node("caller", label=".use()") - g.add_edge("caller", "svc", relation="references") - hits = {h.node_id for h in affected_nodes(g, "svc", depth=2)} - assert "caller" in hits +from tests.native_helpers import graph_from_payload + + +def test_class_seed_reaches_method_bound_caller_without_reporting_member(): + graph = graph_from_payload( + [ + {"id": "class", "label": "Service", "source_file": "service.py"}, + {"id": "method", "label": "Service.run", "source_file": "service.py"}, + {"id": "caller", "label": "Controller", "source_file": "controller.py"}, + ], + [ + {"source": "class", "target": "method", "relation": "contains"}, + {"source": "caller", "target": "method", "relation": "calls"}, + ], + kind="digraph", + ) + hits = affected_nodes(graph, "class", relations={"calls", "contains"}, depth=2) + assert "caller" in {row.node_id for row in hits} + assert "method" not in {row.node_id for row in hits} diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 7bff432cf..ed20bd05b 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -1,736 +1,135 @@ -"""Tests for analyze.py.""" -import json -import networkx as nx -import pytest -from pathlib import Path -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.analyze import god_nodes, surprising_connections, _is_concept_node, graph_diff, _surprise_score, _file_category, _is_json_key_node, find_import_cycles, suggest_questions -from graphify.extract import _make_id - -FIXTURES = Path(__file__).parent / "fixtures" - - -def make_graph(): - return build_from_json(json.loads((FIXTURES / "extraction.json").read_text())) - - -def test_god_nodes_returns_list(): - G = make_graph() - result = god_nodes(G, top_n=3) - assert isinstance(result, list) - assert len(result) <= 3 - - -def test_god_nodes_sorted_by_degree(): - G = make_graph() - result = god_nodes(G, top_n=10) - degrees = [r["degree"] for r in result] - assert degrees == sorted(degrees, reverse=True) - - -def test_god_nodes_have_required_keys(): - G = make_graph() - result = god_nodes(G, top_n=1) - assert "id" in result[0] - assert "label" in result[0] - assert "degree" in result[0] - - -def test_surprising_connections_cross_source_multi_file(): - """Multi-file graph: should find cross-file edges between real entities.""" - G = make_graph() - communities = cluster(G) - surprises = surprising_connections(G, communities) - assert len(surprises) > 0 - for s in surprises: - assert s["source_files"][0] != s["source_files"][1] - - -def test_surprising_connections_excludes_concept_nodes(): - """Concept nodes (empty source_file) must not appear in surprises.""" - G = make_graph() - # Add a concept node with empty source_file - G.add_node("concept_x", label="Abstract Concept", file_type="document", source_file="") - G.add_edge("n_transformer", "concept_x", relation="relates_to", - confidence="INFERRED", source_file="", weight=0.5) - communities = cluster(G) - surprises = surprising_connections(G, communities) - labels = [s["source"] for s in surprises] + [s["target"] for s in surprises] - assert "Abstract Concept" not in labels - - -def test_surprising_connections_single_file_uses_community_bridges(): - """Single-file graph: should return cross-community edges, not empty list.""" - G = nx.Graph() - # Build a graph with 2 clear communities + 1 bridge edge - for i in range(5): - G.add_node(f"a{i}", label=f"A{i}", file_type="code", source_file="single.py", - source_location=f"L{i}") - for i in range(5): - G.add_node(f"b{i}", label=f"B{i}", file_type="code", source_file="single.py", - source_location=f"L{i+10}") - # Dense intra-community edges - for i in range(4): - G.add_edge(f"a{i}", f"a{i+1}", relation="calls", confidence="EXTRACTED", - source_file="single.py", weight=1.0) - for i in range(4): - G.add_edge(f"b{i}", f"b{i+1}", relation="calls", confidence="EXTRACTED", - source_file="single.py", weight=1.0) - # One cross-community bridge - G.add_edge("a4", "b0", relation="references", confidence="INFERRED", - source_file="single.py", weight=0.5) - - communities = cluster(G) - surprises = surprising_connections(G, communities) - # Should find at least the bridge edge - assert len(surprises) > 0 - - -def test_surprising_connections_ambiguous_scores_higher_than_extracted(): - """AMBIGUOUS edge should score higher than an otherwise identical EXTRACTED edge.""" - G = nx.Graph() - for nid, label, src in [ - ("a", "Alpha", "repo1/model.py"), - ("b", "Beta", "repo2/train.py"), - ("c", "Gamma", "repo1/data.py"), - ("d", "Delta", "repo2/eval.py"), - ]: - G.add_node(nid, label=label, source_file=src, file_type="code") - G.add_edge("a", "b", relation="calls", confidence="AMBIGUOUS", weight=1.0, source_file="repo1/model.py") - G.add_edge("c", "d", relation="calls", confidence="EXTRACTED", weight=1.0, source_file="repo1/data.py") - communities = {0: ["a", "c"], 1: ["b", "d"]} - nc = {"a": 0, "c": 0, "b": 1, "d": 1} - score_amb, _ = _surprise_score(G, "a", "b", G.edges["a", "b"], nc, "repo1/model.py", "repo2/train.py") - score_ext, _ = _surprise_score(G, "c", "d", G.edges["c", "d"], nc, "repo1/data.py", "repo2/eval.py") - assert score_amb > score_ext - - -def test_surprise_score_accepts_precomputed_degrees(): - G = nx.Graph() - for nid, label, src in [ - ("hub", "Hub", "repo1/hub.py"), - ("leaf", "Leaf", "repo2/leaf.py"), - ("n1", "N1", "repo1/n1.py"), - ("n2", "N2", "repo1/n2.py"), - ("n3", "N3", "repo1/n3.py"), - ("n4", "N4", "repo1/n4.py"), - ]: - G.add_node(nid, label=label, source_file=src, file_type="code") - for node in ("leaf", "n1", "n2", "n3", "n4"): - G.add_edge("hub", node, relation="calls", confidence="EXTRACTED", weight=1.0) - - nc = {"hub": 0, "leaf": 1} - edge = G.edges["hub", "leaf"] - args = (G, "hub", "leaf", edge, nc, "repo1/hub.py", "repo2/leaf.py") - - assert _surprise_score(*args) == _surprise_score(*args, dict(G.degree())) - - -def test_surprising_connections_cross_type_scores_higher(): - """Code↔paper edge should score higher than code↔code edge.""" - G = nx.Graph() - for nid, label, src in [ - ("a", "Transformer", "code/model.py"), - ("b", "FlashAttn", "papers/flash.pdf"), - ("c", "Trainer", "code/train.py"), - ("d", "Dataset", "code/data.py"), - ]: - G.add_node(nid, label=label, source_file=src, file_type="code") - G.add_edge("a", "b", relation="references", confidence="EXTRACTED", weight=1.0, source_file="code/model.py") - G.add_edge("c", "d", relation="calls", confidence="EXTRACTED", weight=1.0, source_file="code/train.py") - nc = {"a": 0, "b": 1, "c": 0, "d": 0} - score_cross, reasons_cross = _surprise_score(G, "a", "b", G.edges["a", "b"], nc, "code/model.py", "papers/flash.pdf") - score_same, _ = _surprise_score(G, "c", "d", G.edges["c", "d"], nc, "code/train.py", "code/data.py") - assert score_cross > score_same - assert any("code" in r and "paper" in r for r in reasons_cross) - - -def _make_cross_lang_graph(): - """Helper: Python node in backend/, TypeScript node in frontend/, different communities.""" - G = nx.Graph() - G.add_node("py_auth", label="AuthError", source_file="backend/auth.py", file_type="code") - G.add_node("ts_member", label="Member", source_file="frontend/types.ts", file_type="code") - G.add_node("py_a", label="ServiceA", source_file="backend/service.py", file_type="code") - G.add_node("py_b", label="ServiceB", source_file="backend/utils.py", file_type="code") - return G - - -def test_cross_language_inferred_calls_suppressed(): - """Cross-language INFERRED calls edge should score lower than same-language EXTRACTED.""" - G = _make_cross_lang_graph() - G.add_edge("py_auth", "ts_member", relation="calls", confidence="INFERRED", - weight=0.8, source_file="backend/auth.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="backend/service.py") - nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} - score_cross, _ = _surprise_score(G, "py_auth", "ts_member", - G.edges["py_auth", "ts_member"], nc, - "backend/auth.py", "frontend/types.ts") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "backend/service.py", "backend/utils.py") - assert score_cross <= score_same - - -def test_cross_language_inferred_uses_suppressed(): - """Cross-language INFERRED uses edge (the exact rsl-siege-manager false positive) should be suppressed.""" - G = _make_cross_lang_graph() - G.add_edge("py_auth", "ts_member", relation="uses", confidence="INFERRED", - weight=0.8, source_file="backend/auth.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="backend/service.py") - nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} - score_cross, _ = _surprise_score(G, "py_auth", "ts_member", - G.edges["py_auth", "ts_member"], nc, - "backend/auth.py", "frontend/types.ts") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "backend/service.py", "backend/utils.py") - assert score_cross <= score_same - - -def test_cross_language_semantically_similar_not_suppressed(): - """`semantically_similar_to` across languages is a genuine insight — must not be suppressed.""" - G = _make_cross_lang_graph() - G.add_edge("py_auth", "ts_member", relation="semantically_similar_to", - confidence="INFERRED", weight=0.85, source_file="backend/auth.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="backend/service.py") - nc = {"py_auth": 0, "ts_member": 1, "py_a": 0, "py_b": 0} - score_sem, _ = _surprise_score(G, "py_auth", "ts_member", - G.edges["py_auth", "ts_member"], nc, - "backend/auth.py", "frontend/types.ts") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "backend/service.py", "backend/utils.py") - assert score_sem > score_same - - -def test_same_language_inferred_calls_not_suppressed(): - """INFERRED calls within the same language family must not be affected.""" - G = nx.Graph() - G.add_node("py_a", label="ModuleA", source_file="src/a.py", file_type="code") - G.add_node("py_b", label="ModuleB", source_file="src/b.py", file_type="code") - G.add_node("py_c", label="ModuleC", source_file="src/c.py", file_type="code") - G.add_node("py_d", label="ModuleD", source_file="src/d.py", file_type="code") - G.add_edge("py_a", "py_b", relation="calls", confidence="INFERRED", - weight=0.8, source_file="src/a.py") - G.add_edge("py_c", "py_d", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/c.py") - nc = {"py_a": 0, "py_b": 1, "py_c": 0, "py_d": 1} - score_inf, _ = _surprise_score(G, "py_a", "py_b", G.edges["py_a", "py_b"], nc, - "src/a.py", "src/b.py") - score_ext, _ = _surprise_score(G, "py_c", "py_d", G.edges["py_c", "py_d"], nc, - "src/c.py", "src/d.py") - assert score_inf > score_ext - - -def test_cross_language_extracted_calls_not_suppressed(): - """EXTRACTED cross-language edges are real structural facts — must not be penalised.""" - G = _make_cross_lang_graph() - G.add_edge("py_auth", "ts_member", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="backend/auth.py") - nc = {"py_auth": 0, "ts_member": 1} - score, _ = _surprise_score(G, "py_auth", "ts_member", - G.edges["py_auth", "ts_member"], nc, - "backend/auth.py", "frontend/types.ts") - assert score >= 1 - - -def test_surprising_connections_have_why_field(): - G = make_graph() - communities = cluster(G) - for s in surprising_connections(G, communities): - assert "why" in s - assert isinstance(s["why"], str) - assert len(s["why"]) > 0 - - -def test_file_category(): - assert _file_category("model.py") == "code" - assert _file_category("flash.pdf") == "paper" - assert _file_category("diagram.png") == "image" - assert _file_category("notes.md") == "doc" - # Languages added in later releases — would misclassify as "doc" without detect.py import - assert _file_category("app.swift") == "code" - assert _file_category("plugin.lua") == "code" - assert _file_category("build.zig") == "code" - assert _file_category("deploy.ps1") == "code" - assert _file_category("server.ex") == "code" - assert _file_category("component.jsx") == "code" - assert _file_category("analysis.jl") == "code" - assert _file_category("view.m") == "code" - - -def test_is_concept_node_empty_source(): - G = nx.Graph() - G.add_node("c1", source_file="") - assert _is_concept_node(G, "c1") is True - - -def test_is_concept_node_real_file(): - G = nx.Graph() - G.add_node("n1", source_file="model.py") - assert _is_concept_node(G, "n1") is False - - -def test_surprising_connections_have_required_keys(): - G = make_graph() - communities = cluster(G) - for s in surprising_connections(G, communities): - assert "source" in s - assert "target" in s - assert "source_files" in s - assert "confidence" in s - - -# --- graph_diff tests --- - -def _make_simple_graph(nodes, edges): - """Helper: build a small nx.Graph from node/edge specs.""" - G = nx.Graph() - for node_id, label in nodes: - G.add_node(node_id, label=label, source_file="test.py") - for src, tgt, rel, conf in edges: - G.add_edge(src, tgt, relation=rel, confidence=conf) - return G - - -def test_graph_diff_new_nodes(): - G_old = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta")], []) - G_new = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")], []) - diff = graph_diff(G_old, G_new) - assert len(diff["new_nodes"]) == 1 - assert diff["new_nodes"][0]["id"] == "n3" - assert diff["new_nodes"][0]["label"] == "Gamma" - assert diff["removed_nodes"] == [] - assert "1 new node" in diff["summary"] - - -def test_graph_diff_removed_nodes(): - G_old = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")], []) - G_new = _make_simple_graph([("n1", "Alpha"), ("n2", "Beta")], []) - diff = graph_diff(G_old, G_new) - assert diff["new_nodes"] == [] - assert len(diff["removed_nodes"]) == 1 - assert diff["removed_nodes"][0]["id"] == "n3" - assert "removed" in diff["summary"] - - -def test_graph_diff_new_edges(): - nodes = [("n1", "Alpha"), ("n2", "Beta"), ("n3", "Gamma")] - G_old = _make_simple_graph(nodes, [("n1", "n2", "calls", "EXTRACTED")]) - G_new = _make_simple_graph( - nodes, - [("n1", "n2", "calls", "EXTRACTED"), ("n2", "n3", "uses", "INFERRED")], +"""Analysis behavior on real embedded-Helix snapshots.""" + +from graphify.analyze import ( + _file_category, + _is_concept_node, + _is_json_key_node, + _surprise_score, + find_import_cycles, + god_nodes, + graph_diff, + suggest_questions, + surprising_connections, +) +from graphify.helix.access import first_edge_attributes +from tests.native_helpers import graph_from_payload, triangle + + +def _graph(): + return graph_from_payload( + [ + {"id": "hub", "label": "Hub", "file_type": "code", "source_file": "src/hub.py"}, + {"id": "left", "label": "Left", "file_type": "code", "source_file": "src/left.py"}, + {"id": "right", "label": "Right", "file_type": "code", "source_file": "web/right.ts"}, + {"id": "doc", "label": "Design", "file_type": "document", "source_file": "docs/design.md"}, + ], + [ + {"source": "hub", "target": "left", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "hub", "target": "right", "relation": "references", "confidence": "INFERRED"}, + {"source": "hub", "target": "doc", "relation": "documents", "confidence": "INFERRED"}, + ], + kind="digraph", ) - diff = graph_diff(G_old, G_new) - assert len(diff["new_edges"]) == 1 - new_edge = diff["new_edges"][0] - assert new_edge["relation"] == "uses" - assert new_edge["confidence"] == "INFERRED" - assert diff["removed_edges"] == [] - assert "new edge" in diff["summary"] - - -def test_graph_diff_empty_diff(): - nodes = [("n1", "Alpha"), ("n2", "Beta")] - edges = [("n1", "n2", "calls", "EXTRACTED")] - G_old = _make_simple_graph(nodes, edges) - G_new = _make_simple_graph(nodes, edges) - diff = graph_diff(G_old, G_new) - assert diff["new_nodes"] == [] - assert diff["removed_nodes"] == [] - assert diff["new_edges"] == [] - assert diff["removed_edges"] == [] - assert diff["summary"] == "no changes" - - -# --- code↔doc INFERRED suppression tests --- - -def _make_code_doc_graph(): - G = nx.Graph() - G.add_node("py_fn", label="ProcessData", source_file="src/processor.py", file_type="code") - G.add_node("md_doc", label="README Section", source_file="docs/readme.md", file_type="document") - G.add_node("py_a", label="ServiceA", source_file="src/service.py", file_type="code") - G.add_node("py_b", label="ServiceB", source_file="src/utils.py", file_type="code") - return G - - -def test_code_doc_inferred_calls_suppressed(): - """Code→doc INFERRED calls edge should score lower than same-language EXTRACTED.""" - G = _make_code_doc_graph() - G.add_edge("py_fn", "md_doc", relation="calls", confidence="INFERRED", - weight=0.8, source_file="src/processor.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/service.py") - nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} - score_noise, _ = _surprise_score(G, "py_fn", "md_doc", - G.edges["py_fn", "md_doc"], nc, - "src/processor.py", "docs/readme.md") - score_real, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/service.py", "src/utils.py") - assert score_noise <= score_real -def test_code_doc_inferred_uses_suppressed(): - """Code→doc INFERRED uses edge should score lower than same-language EXTRACTED.""" - G = _make_code_doc_graph() - G.add_edge("py_fn", "md_doc", relation="uses", confidence="INFERRED", - weight=0.8, source_file="src/processor.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/service.py") - nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} - score_noise, _ = _surprise_score(G, "py_fn", "md_doc", - G.edges["py_fn", "md_doc"], nc, - "src/processor.py", "docs/readme.md") - score_real, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/service.py", "src/utils.py") - assert score_noise <= score_real - - -def test_code_doc_extracted_calls_not_suppressed(): - """EXTRACTED code↔doc edges are real facts — must not be penalised.""" - G = _make_code_doc_graph() - G.add_edge("py_fn", "md_doc", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/processor.py") - nc = {"py_fn": 0, "md_doc": 1} - score, _ = _surprise_score(G, "py_fn", "md_doc", - G.edges["py_fn", "md_doc"], nc, - "src/processor.py", "docs/readme.md") - assert score >= 1 - - -def test_code_doc_inferred_semantically_similar_not_suppressed(): - """`semantically_similar_to` across code↔doc is explicit LLM insight — must not be suppressed.""" - G = _make_code_doc_graph() - G.add_edge("py_fn", "md_doc", relation="semantically_similar_to", - confidence="INFERRED", weight=0.85, source_file="src/processor.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/service.py") - nc = {"py_fn": 0, "md_doc": 1, "py_a": 0, "py_b": 0} - score_sem, _ = _surprise_score(G, "py_fn", "md_doc", - G.edges["py_fn", "md_doc"], nc, - "src/processor.py", "docs/readme.md") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/service.py", "src/utils.py") - assert score_sem > score_same - - -def test_code_unknown_extension_inferred_calls_suppressed(): - """_file_category falls back to 'doc' for unknown extensions, so INFERRED - calls/uses to unknown-extension files are suppressed the same as code↔doc.""" - assert _file_category("vendor/random.xyz") == "doc" - G = nx.Graph() - G.add_node("py_fn", label="Handler", source_file="src/handler.py", file_type="code") - G.add_node("unk", label="Handler", source_file="vendor/unknown.xyz", file_type="document") - G.add_node("py_a", label="A", source_file="src/a.py", file_type="code") - G.add_node("py_b", label="B", source_file="src/b.py", file_type="code") - G.add_edge("py_fn", "unk", relation="calls", confidence="INFERRED", - weight=0.8, source_file="src/handler.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/a.py") - nc = {"py_fn": 0, "unk": 1, "py_a": 0, "py_b": 0} - score_unk, _ = _surprise_score(G, "py_fn", "unk", - G.edges["py_fn", "unk"], nc, - "src/handler.py", "vendor/unknown.xyz") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/a.py", "src/b.py") - assert score_unk <= score_same - - -def test_code_paper_inferred_calls_not_suppressed(): - """Code↔paper INFERRED calls should still surface — it is a meaningful link.""" - G = nx.Graph() - G.add_node("py_model", label="Transformer", source_file="src/model.py", file_type="code") - G.add_node("pdf_paper", label="Attention Is All You Need", source_file="papers/vaswani.pdf", - file_type="paper") - G.add_node("py_a", label="ServiceA", source_file="src/service.py", file_type="code") - G.add_node("py_b", label="ServiceB", source_file="src/utils.py", file_type="code") - G.add_edge("py_model", "pdf_paper", relation="calls", confidence="INFERRED", - weight=0.8, source_file="src/model.py") - G.add_edge("py_a", "py_b", relation="calls", confidence="EXTRACTED", - weight=1.0, source_file="src/service.py") - nc = {"py_model": 0, "pdf_paper": 1, "py_a": 0, "py_b": 1} - score_cross, _ = _surprise_score(G, "py_model", "pdf_paper", - G.edges["py_model", "pdf_paper"], nc, - "src/model.py", "papers/vaswani.pdf") - score_same, _ = _surprise_score(G, "py_a", "py_b", - G.edges["py_a", "py_b"], nc, - "src/service.py", "src/utils.py") - assert score_cross > score_same - - -# --- JSON key node filtering tests --- - -def test_is_json_key_node_noise_label(): - G = nx.Graph() - G.add_node("j1", label="name", source_file="schema.json") - assert _is_json_key_node(G, "j1") is True - - -def test_is_json_key_node_non_json_file(): - G = nx.Graph() - G.add_node("n1", label="name", source_file="model.py") - assert _is_json_key_node(G, "n1") is False - - -# --- npm dep-block key god-node filtering tests --- - -@pytest.mark.parametrize("dep_key", [ - "dependencies", - "devDependencies", - "peerDependencies", - "optionalDependencies", - "bundledDependencies", -]) -def test_god_nodes_excludes_npm_dep_block_keys(dep_key: str) -> None: - """npm package.json dep-block keys must be filtered from god_nodes output. - - Constructs a small graph with one node labelled with an npm dep-block key - (sourced from a .json file) and one real-domain node that has high degree. - Asserts that god_nodes() excludes the dep-block node even when it has the - highest degree, while the real-domain node is included. - - Args: - dep_key: The npm dependency-block key label to test (parametrized). - """ - G = nx.Graph() - # Real-domain node with a realistic source file. - G.add_node( - "real_node", - label="AuthService", - source_file="src/auth.py", - file_type="code", - source_location="L1", +def test_analysis_runs_on_native_snapshot(tmp_path): + graph = triangle(tmp_path).graph + communities = {0: ["a", "b"], 1: ["c"]} + assert god_nodes(graph) + assert surprising_connections(graph, communities) + assert suggest_questions(graph, communities, {0: "Core", 1: "Leaf"}) + + +def test_god_nodes_are_ranked_and_structured(): + result = god_nodes(_graph(), top_n=3) + assert result[0]["id"] == "hub" + assert result[0]["degree"] == 3 + assert {"id", "label", "degree"} <= set(result[0]) + + +def test_surprises_exclude_concepts_and_keep_cross_file_edges(): + graph = graph_from_payload( + [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, + {"id": "concept", "label": "Concept", "file_type": "document", "source_file": ""}, + ], + [ + {"source": "a", "target": "b", "relation": "references", "confidence": "INFERRED"}, + {"source": "a", "target": "concept", "relation": "references", "confidence": "INFERRED"}, + ], ) - # npm dep-block key node — sourced from a JSON file so _is_json_key_node fires. - G.add_node( - "dep_node", - label=dep_key, - source_file="frontend/package.json", - file_type="code", - source_location="L1", + result = surprising_connections(graph, {0: ["a", "concept"], 1: ["b"]}) + labels = {item["source"] for item in result} | {item["target"] for item in result} + assert result and "Concept" not in labels + assert result[0]["source_files"][0] != result[0]["source_files"][1] + assert result[0]["why"] + + +def test_surprise_scoring_retains_confidence_and_language_rules(): + graph = graph_from_payload( + [ + {"id": "py", "label": "Auth", "source_file": "api/auth.py", "file_type": "code"}, + {"id": "ts", "label": "Member", "source_file": "web/types.ts", "file_type": "code"}, + {"id": "doc", "label": "Auth", "source_file": "docs/auth.md", "file_type": "document"}, + ], + [ + {"source": "py", "target": "ts", "relation": "calls", "confidence": "INFERRED"}, + {"source": "py", "target": "doc", "relation": "semantically_similar_to", "confidence": "INFERRED"}, + ], ) - # Wire up enough edges so dep_node has high degree — it would be a god-node - # without the filter. - for i in range(20): - peer = f"pkg_{i}" - G.add_node( - peer, - label=f"package-{i}", - source_file="frontend/package.json", - file_type="code", - source_location=f"L{i + 2}", - ) - G.add_edge( - "dep_node", - peer, - relation="contains", - confidence="EXTRACTED", - source_file="frontend/package.json", - weight=1.0, - ) - # Give real_node a couple of edges too. - G.add_edge( - "real_node", - "dep_node", - relation="imports", - confidence="EXTRACTED", - source_file="src/auth.py", - weight=1.0, + communities = {"py": 0, "ts": 1, "doc": 1} + cross, _ = _surprise_score( + graph, "py", "ts", first_edge_attributes(graph, "py", "ts"), communities, + "api/auth.py", "web/types.ts", ) - - result = god_nodes(G, top_n=10) - result_ids = [r["id"] for r in result] - - assert "dep_node" not in result_ids, ( - f"god_nodes() should filter npm dep-block key '{dep_key}' " - f"but it appeared in the result: {result}" + semantic, _ = _surprise_score( + graph, "py", "doc", first_edge_attributes(graph, "py", "doc"), communities, + "api/auth.py", "docs/auth.md", ) - assert "real_node" in result_ids, ( - f"god_nodes() should include real-domain node 'AuthService' " - f"but it was absent: {result}" + assert semantic > cross + + +def test_node_noise_helpers_use_native_attributes(): + graph = graph_from_payload([ + {"id": "concept", "source_file": ""}, + {"id": "json", "label": "dependencies", "source_file": "package.json"}, + {"id": "real", "label": "Auth", "source_file": "auth.py"}, + ]) + assert _is_concept_node(graph, "concept") + assert _is_json_key_node(graph, "json") + assert not _is_json_key_node(graph, "real") + assert _file_category("paper.pdf") == "paper" + + +def test_graph_diff_native_nodes_and_edges(): + old = graph_from_payload([ + {"id": "a", "label": "A"}, {"id": "b", "label": "B"}, + ], [{"source": "a", "target": "b", "relation": "calls"}]) + new = graph_from_payload([ + {"id": "a", "label": "A"}, {"id": "c", "label": "C"}, + ], [{"source": "a", "target": "c", "relation": "uses"}]) + result = graph_diff(old, new) + assert [row["id"] for row in result["new_nodes"]] == ["c"] + assert [row["id"] for row in result["removed_nodes"]] == ["b"] + assert result["new_edges"][0]["relation"] == "uses" + assert result["removed_edges"][0]["relation"] == "calls" + + +def test_import_cycles_support_self_loops_and_length_limit(): + graph = graph_from_payload( + [ + {"id": "a", "label": "A", "source_file": "a.py"}, + {"id": "b", "label": "B", "source_file": "b.py"}, + {"id": "c", "label": "C", "source_file": "c.py"}, + ], + [ + {"source": "a", "target": "b", "relation": "imports_from", "source_file": "a.py"}, + {"source": "b", "target": "a", "relation": "imports_from", "source_file": "b.py"}, + {"source": "c", "target": "c", "relation": "imports_from", "source_file": "c.py"}, + ], + kind="digraph", ) - - -def test_is_json_key_node_real_label(): - G = nx.Graph() - G.add_node("j2", label="UserProfile", source_file="schema.json") - assert _is_json_key_node(G, "j2") is False - - -def test_god_nodes_excludes_json_noise(): - """god_nodes must not return generic JSON key nodes like 'name' or 'id'.""" - G = nx.Graph() - # Add many edges to a real node - G.add_node("real", label="AuthService", source_file="src/auth.py") - # Add a noisy JSON key node with high degree - G.add_node("json_name", label="name", source_file="schema.json") - for i in range(8): - n = f"peer{i}" - G.add_node(n, label=f"Peer{i}", source_file=f"src/peer{i}.py") - G.add_edge("json_name", n) - G.add_edge("real", n) - result = god_nodes(G, top_n=10) - labels = [r["label"] for r in result] - assert "name" not in labels - assert "AuthService" in labels - - -def test_god_nodes_filter_is_case_insensitive(): - """JSON-key filter must match regardless of label casing.""" - G = nx.Graph() - G.add_node("real", label="RealAbstraction", source_file="libs/real.py") - for i in range(3): - G.add_node(f"peer{i}", label=f"P{i}", source_file=f"src/p{i}.py") - G.add_edge("real", f"peer{i}") - for variant in ("Start", "START", "Name", "ID"): - nid = f"json_{variant.lower()}" - G.add_node(nid, label=variant, source_file="testhelpers/data.json") - for i in range(15): - t = f"{nid}_t{i}" - G.add_node(t, label=f"X{i}", source_file="testhelpers/data.json") - G.add_edge(t, nid) - result = god_nodes(G, top_n=10) - labels = [r["label"] for r in result] - for variant in ("Start", "START", "Name", "ID"): - assert variant not in labels, f"`{variant}` should be filtered as JSON-key noise" - - -def test_suggest_questions_excludes_rationale_nodes_from_isolated_count(): - G = nx.Graph() - G.add_node("service", label="Service", file_type="code", source_file="service.py") - G.add_node("reason", label="Explains service", file_type="rationale", source_file="service.py") - - questions = suggest_questions(G, communities={}, community_labels={}, top_n=10) - isolated = next(question for question in questions if question["type"] == "isolated_nodes") - - assert isolated["why"].startswith("1 weakly-connected node") - assert "`Service`" in isolated["question"] - assert "Explains service" not in isolated["question"] - - -# ── find_import_cycles tests ────────────────────────────────────────────────── - - -def _make_file_node(path: str) -> tuple[str, dict]: - """Create a graph node resembling real graphify schema.""" - nid = _make_id(path) - return nid, {"label": Path(path).name, "source_file": path, "file_type": "code"} - - -def _make_cycle_graph_directed() -> nx.DiGraph: - G = nx.DiGraph() - - a_id, a = _make_file_node("src/a.ts") - b_id, b = _make_file_node("src/b.ts") - c_id, c = _make_file_node("src/c.ts") - d_id, d = _make_file_node("src/d.ts") - ext_id = _make_id("react") - - G.add_node(a_id, **a) - G.add_node(b_id, **b) - G.add_node(c_id, **c) - G.add_node(d_id, **d) - # External-like node (no source_file): must be skipped safely. - G.add_node(ext_id, label="react", file_type="code") - - # 2-cycle: a <-> b - G.add_edge(a_id, b_id, relation="imports_from", source_file="src/a.ts", confidence="EXTRACTED") - G.add_edge(b_id, a_id, relation="imports_from", source_file="src/b.ts", confidence="EXTRACTED") - - # 3-cycle: b -> c -> d -> b - G.add_edge(b_id, c_id, relation="imports_from", source_file="src/b.ts", confidence="EXTRACTED") - G.add_edge(c_id, d_id, relation="imports_from", source_file="src/c.ts", confidence="EXTRACTED") - G.add_edge(d_id, b_id, relation="imports_from", source_file="src/d.ts", confidence="EXTRACTED") - - # Self-loop: c imports itself - G.add_edge(c_id, c_id, relation="imports_from", source_file="src/c.ts", confidence="EXTRACTED") - - # Mixed edge types: must not bleed into cycle graph - G.add_edge(a_id, ext_id, relation="calls", source_file="src/a.ts", confidence="INFERRED") - G.add_edge(a_id, ext_id, relation="contains", source_file="src/a.ts", confidence="EXTRACTED") - - # Edge whose target has no source_file: must be skipped, no garbage label fallback - G.add_edge(a_id, ext_id, relation="imports_from", source_file="src/a.ts", confidence="EXTRACTED") - - return G - - -def test_find_import_cycles_returns_structured_records(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G) - assert isinstance(cycles, list) - assert cycles - assert isinstance(cycles[0], dict) - assert "cycle" in cycles[0] - assert "length" in cycles[0] - assert "why" in cycles[0] - - -def test_find_import_cycles_detects_2_and_3_cycles(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G) - cycle_sets = [set(c["cycle"]) for c in cycles] - assert any({"src/a.ts", "src/b.ts"}.issubset(s) for s in cycle_sets) - assert any({"src/b.ts", "src/c.ts", "src/d.ts"}.issubset(s) for s in cycle_sets) - - -def test_find_import_cycles_includes_self_loop_cycle(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G) - assert any(c["cycle"] == ["src/c.ts"] and c["length"] == 1 for c in cycles) - - -def test_find_import_cycles_respects_max_cycle_length(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G, max_cycle_length=2) - assert all(c["length"] <= 2 for c in cycles) - - -def test_find_import_cycles_skips_nodes_without_source_file(): - G = _make_cycle_graph_directed() - cycles = find_import_cycles(G) - flat = " ".join(" ".join(c["cycle"]) for c in cycles) - assert "react" not in flat - - -def test_find_import_cycles_handles_undirected_graph_input(): - Gd = _make_cycle_graph_directed() - Gu = nx.Graph() - Gu.add_nodes_from(Gd.nodes(data=True)) - Gu.add_edges_from(Gd.edges(data=True)) - cycles = find_import_cycles(Gu) - assert cycles # should still resolve orientation via edge.source_file - - -def test_find_import_cycles_ignores_non_import_relations(): - G = nx.DiGraph() - a_id, a = _make_file_node("src/a.ts") - b_id, b = _make_file_node("src/b.ts") - G.add_node(a_id, **a) - G.add_node(b_id, **b) - # Bidirectional non-import edges should not be considered a dependency cycle. - G.add_edge(a_id, b_id, relation="calls", source_file="src/a.ts", confidence="INFERRED") - G.add_edge(b_id, a_id, relation="contains", source_file="src/b.ts", confidence="EXTRACTED") - assert find_import_cycles(G) == [] - - -def test_find_import_cycles_empty_graph(): - assert find_import_cycles(nx.DiGraph()) == [] - - -def test_find_import_cycles_no_cycles(): - G = nx.DiGraph() - x_id, x = _make_file_node("x.ts") - y_id, y = _make_file_node("y.ts") - G.add_node(x_id, **x) - G.add_node(y_id, **y) - G.add_edge(x_id, y_id, relation="imports_from", source_file="x.ts", confidence="EXTRACTED") - assert find_import_cycles(G) == [] + cycles = find_import_cycles(graph, max_cycle_length=2) + assert {tuple(item["cycle"]) for item in cycles} + assert any(len(item["cycle"]) == 1 for item in cycles) diff --git a/tests/test_atomic_writes.py b/tests/test_atomic_writes.py index 968cb0b96..f8971eb53 100644 --- a/tests/test_atomic_writes.py +++ b/tests/test_atomic_writes.py @@ -1,4 +1,4 @@ -"""Tests for atomic JSON writes (graph.json / manifest.json). +"""Tests for atomic writes used by presentation artifacts and manifests. A crash, kill, or disk-full mid-write must not leave a truncated/corrupt file that a later load chokes on. `write_text_atomic` writes a temp file in the same @@ -13,15 +13,15 @@ def test_write_text_atomic_writes_and_leaves_no_tmp(tmp_path): - p = tmp_path / "out" / "graph.json" # parent doesn't exist yet + p = tmp_path / "out" / "artifact.json" # parent doesn't exist yet write_text_atomic(p, '{"a": 1}') assert json.loads(p.read_text()) == {"a": 1} # No leftover temp file in the target directory. - assert [x.name for x in p.parent.iterdir()] == ["graph.json"] + assert [x.name for x in p.parent.iterdir()] == ["artifact.json"] def test_write_text_atomic_preserves_existing_on_failure(tmp_path, monkeypatch): - p = tmp_path / "graph.json" + p = tmp_path / "artifact.json" p.write_text("original", encoding="utf-8") def boom(src, dst): @@ -33,12 +33,12 @@ def boom(src, dst): # The original file is intact and the temp file was cleaned up. assert p.read_text() == "original" - assert sorted(x.name for x in tmp_path.iterdir()) == ["graph.json"] + assert sorted(x.name for x in tmp_path.iterdir()) == ["artifact.json"] def test_write_text_atomic_preserves_existing_mode(tmp_path): # An atomic replace must not tighten a 0644 file to mkstemp's 0600 default. - p = tmp_path / "graph.json" + p = tmp_path / "artifact.json" p.write_text("{}", encoding="utf-8") os.chmod(p, 0o644) write_text_atomic(p, '{"x": 1}') @@ -47,7 +47,7 @@ def test_write_text_atomic_preserves_existing_mode(tmp_path): def test_write_text_atomic_new_file_respects_umask(tmp_path): # A brand-new file must land at the umask default (e.g. 0644), NOT mkstemp's - # 0600 — otherwise every fresh graph.json would be owner-only. + # 0600 — otherwise every fresh presentation artifact would be owner-only. p = tmp_path / "new.json" write_text_atomic(p, "{}") umask = os.umask(0) @@ -56,7 +56,7 @@ def test_write_text_atomic_new_file_respects_umask(tmp_path): def test_write_text_atomic_writes_through_symlink(tmp_path): - # Shared-output setups symlink graph.json to shared storage; the atomic write + # Shared-output setups may symlink an artifact to shared storage; the atomic write # must update the target and keep the link, not replace it with a real file. target = tmp_path / "real.json" target.write_text("old", encoding="utf-8") @@ -76,20 +76,6 @@ def test_write_json_atomic_roundtrip(tmp_path): assert not any(name.name.endswith(".tmp") for name in tmp_path.iterdir()) -def test_to_json_writes_atomically_no_tmp_leftover(tmp_path): - import networkx as nx - from graphify.export import to_json - - G = nx.Graph() - G.add_node("a", label="a", file_type="code") - G.add_node("b", label="b", file_type="code") - G.add_edge("a", "b") - out = tmp_path / "graph.json" - assert to_json(G, {}, str(out), force=True) is True - json.loads(out.read_text()) # valid JSON - assert not any(x.name.endswith(".tmp") for x in tmp_path.iterdir()) - - def test_save_manifest_writes_atomically(tmp_path): from graphify.detect import save_manifest @@ -105,7 +91,7 @@ def test_write_text_atomic_windows_permission_fallback(tmp_path, monkeypatch): """On Windows os.replace raises PermissionError when the destination is briefly locked (antivirus, an open reader); the copy-then-delete fallback must still land the new content and leave no temp file.""" - p = tmp_path / "graph.json" + p = tmp_path / "artifact.json" p.write_text("original", encoding="utf-8") real_replace = os.replace @@ -120,7 +106,7 @@ def flaky_replace(src, dst): assert calls["n"] == 1 # the fallback path was actually exercised assert p.read_text() == "new-content" - assert sorted(x.name for x in tmp_path.iterdir()) == ["graph.json"] + assert sorted(x.name for x in tmp_path.iterdir()) == ["artifact.json"] def test_write_json_atomic_ensure_ascii_false_preserves_utf8(tmp_path): diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index b5751adcc..4d90cf858 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -1,183 +1,45 @@ -"""Tests for graphify/benchmark.py.""" -from __future__ import annotations -import json -import pytest -import networkx as nx -from networkx.readwrite import json_graph - -from graphify.benchmark import run_benchmark, print_benchmark, _query_subgraph_tokens, _SAMPLE_QUESTIONS, _safe, _hr - - -def _make_graph() -> nx.Graph: - G = nx.Graph() - G.add_node("n1", label="authentication", source_file="auth.py", source_location="L1", community=0) - G.add_node("n2", label="api_handler", source_file="api.py", source_location="L5", community=0) - G.add_node("n3", label="main_entry", source_file="main.py", source_location="L1", community=1) - G.add_node("n4", label="error_handler", source_file="errors.py", source_location="L1", community=1) - G.add_node("n5", label="database_layer", source_file="db.py", source_location="L1", community=2) - G.add_edge("n1", "n2", relation="calls", confidence="INFERRED") - G.add_edge("n2", "n3", relation="imports", confidence="EXTRACTED") - G.add_edge("n3", "n4", relation="uses", confidence="EXTRACTED") - G.add_edge("n5", "n2", relation="provides", confidence="EXTRACTED") - return G - - -def _write_graph(G: nx.Graph, path) -> None: - data = json_graph.node_link_data(G, edges="links") - path.write_text(json.dumps(data)) - - -# --- _query_subgraph_tokens --- - -def test_query_returns_positive_for_matching_question(): - G = _make_graph() - tokens = _query_subgraph_tokens(G, "how does authentication work") - assert tokens > 0 - -def test_query_returns_zero_for_no_match(): - G = _make_graph() - tokens = _query_subgraph_tokens(G, "xyzzy plugh zorkmid") - assert tokens == 0 - -def test_query_bfs_expands_neighbors(): - G = _make_graph() - # "authentication" matches n1, BFS depth=3 should reach n2, n3, n4 - tokens_deep = _query_subgraph_tokens(G, "authentication", depth=3) - tokens_shallow = _query_subgraph_tokens(G, "authentication", depth=1) - assert tokens_deep >= tokens_shallow - - -def test_query_keeps_short_non_english_terms(): - G = nx.Graph() - G.add_node("frontend", label="前端", source_file="docs/前端.md", source_location="L1", community=0) - tokens = _query_subgraph_tokens(G, "前端", depth=1) - assert tokens > 0 - - -# --- run_benchmark --- - -def test_run_benchmark_returns_reduction(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=10_000) - assert "reduction_ratio" in result - assert result["reduction_ratio"] > 1.0 - -def test_run_benchmark_corpus_tokens_proportional(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - r1 = run_benchmark(str(graph_file), corpus_words=1_000) - r2 = run_benchmark(str(graph_file), corpus_words=10_000) - # corpus_tokens scales linearly with corpus_words (within integer-division rounding) - assert abs(r2["corpus_tokens"] - r1["corpus_tokens"] * 10) <= r1["corpus_tokens"] - -def test_run_benchmark_per_question_list(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=5_000, - questions=["how does authentication work", "what is the main entry"]) - assert len(result["per_question"]) >= 1 - for p in result["per_question"]: - assert "question" in p - assert "query_tokens" in p - assert "reduction" in p - -def test_run_benchmark_estimates_corpus_if_no_words(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=None) - assert result["corpus_words"] > 0 - -def test_run_benchmark_error_on_empty_graph(tmp_path): - G = nx.Graph() - graph_file = tmp_path / "empty.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=1_000) - assert "error" in result - -def test_run_benchmark_includes_node_edge_counts(tmp_path): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=5_000) - assert result["nodes"] == G.number_of_nodes() - assert result["edges"] == G.number_of_edges() - - -# --- print_benchmark --- - -def test_print_benchmark_no_crash(tmp_path, capsys): - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=5_000) +from graphify.benchmark import _estimate_tokens, _query_subgraph_tokens, print_benchmark, run_benchmark +from tests.native_helpers import make_loaded + + +def _loaded(tmp_path): + return make_loaded( + tmp_path, + nodes=[ + {"id": "auth", "label": "Authentication", "source_file": "auth.py"}, + {"id": "token", "label": "Token Validator", "source_file": "token.py"}, + {"id": "main", "label": "Main Entry", "source_file": "main.py"}, + ], + edges=[ + {"source": "auth", "target": "token", "relation": "calls"}, + {"source": "main", "target": "auth", "relation": "uses"}, + ], + ) + + +def test_token_estimate_and_native_query(tmp_path): + loaded = _loaded(tmp_path) + assert _estimate_tokens("abcdefgh") == 2 + assert _query_subgraph_tokens(loaded.graph, "authentication", depth=2) > 0 + assert _query_subgraph_tokens(loaded.graph, "xyzzy plugh") == 0 + + +def test_run_benchmark_uses_native_store(tmp_path): + loaded = _loaded(tmp_path) + result = run_benchmark( + str(loaded.store_path), + corpus_words=10_000, + questions=["authentication", "main entry"], + ) + assert result["nodes"] == 3 and result["edges"] == 2 + assert result["reduction_ratio"] > 1 + assert len(result["per_question"]) == 2 + + +def test_print_benchmark_handles_success_and_error(tmp_path, capsys): + result = run_benchmark(str(_loaded(tmp_path).store_path), corpus_words=1000, + questions=["authentication"]) print_benchmark(result) - out = capsys.readouterr().out - assert "reduction" in out.lower() - assert "x" in out - -def test_print_benchmark_error_message(capsys): - print_benchmark({"error": "test error message"}) - out = capsys.readouterr().out - assert "test error message" in out - - -# --- cp1252 / Windows-console encoding compatibility (regression for #?) --- -# print_benchmark previously crashed on Windows consoles (cp1252) because it -# unconditionally printed U+2500 and U+2192. _safe() falls back to ASCII when -# stdout cannot encode the glyph. - -def test_safe_returns_unicode_when_encodable(): - import io, sys - real_stdout = sys.stdout - try: - sys.stdout = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") - assert _safe("→", "->") == "→" - assert _hr(5) == "─" * 5 - finally: - sys.stdout = real_stdout - -def test_safe_falls_back_when_unencodable(): - import io, sys - real_stdout = sys.stdout - try: - sys.stdout = io.TextIOWrapper(io.BytesIO(), encoding="cp1252") - assert _safe("→", "->") == "->" - assert _hr(5) == "-" * 5 - finally: - sys.stdout = real_stdout - -def test_print_benchmark_survives_cp1252_stdout(tmp_path, monkeypatch, capsys): - """Regression: U+2500 / U+2192 used to crash with UnicodeEncodeError on cp1252.""" - import io, sys - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - result = run_benchmark(str(graph_file), corpus_words=5_000) - - # Replace stdout with a strict cp1252 stream — same behaviour as the - # legacy Windows console that surfaced this bug. - cp1252_stdout = io.TextIOWrapper(io.BytesIO(), encoding="cp1252", errors="strict") - monkeypatch.setattr(sys, "stdout", cp1252_stdout) - print_benchmark(result) # must not raise UnicodeEncodeError - cp1252_stdout.flush() - written = cp1252_stdout.buffer.getvalue().decode("cp1252") - assert "reduction" in written.lower() - # ASCII fallbacks must be present, fancy glyphs must not. - assert "─" not in written - assert "→" not in written - - -def test_run_benchmark_rejects_oversized_graph(monkeypatch, tmp_path): - """#F4: run_benchmark must refuse to read a graph.json that exceeds - the size cap before parsing it into memory.""" - G = _make_graph() - graph_file = tmp_path / "graph.json" - _write_graph(G, graph_file) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(ValueError, match="exceeds"): - run_benchmark(str(graph_file)) + assert "Reduction" in capsys.readouterr().out + print_benchmark({"error": "empty"}) + assert "Benchmark error" in capsys.readouterr().out diff --git a/tests/test_build.py b/tests/test_build.py index 9e470ad6d..b5f737c17 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,1038 +1,156 @@ -import json -from pathlib import Path -import networkx as nx -from networkx.readwrite import json_graph -from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas, dedupe_edges, dedupe_nodes +"""Transient DTO construction and native incremental merge tests.""" -FIXTURES = Path(__file__).parent / "fixtures" +from graphify.build import ( + build, + build_from_extraction, + build_merge, + build_unclustered_extraction, + dedupe_edges, + dedupe_nodes, +) -def test_dedupe_edges_collapses_exact_parallels(): - # #1317: --no-cluster / incremental update concatenate edge lists raw. - edges = [ - {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, - {"source": "a", "target": "b", "relation": "calls", "source_location": "L9"}, # dup - {"source": "a", "target": "b", "relation": "imports"}, # different relation: kept - {"source": "b", "target": "c", "relation": "calls"}, - ] - out = dedupe_edges(edges) - keys = [(e["source"], e["target"], e["relation"]) for e in out] - assert keys == [("a", "b", "calls"), ("a", "b", "imports"), ("b", "c", "calls")] - # first occurrence wins (keeps L1, not L9) - assert out[0]["source_location"] == "L1" +def _attrs(graph, node_id): + return next(node.attributes for node in graph.nodes if node.id == node_id) -def test_dedupe_edges_is_idempotent(): - edges = [ - {"source": "a", "target": "b", "relation": "calls"}, - {"source": "a", "target": "b", "relation": "calls"}, - ] - once = dedupe_edges(edges) - twice = dedupe_edges(once + edges) # simulate a second `update` re-concatenating - assert len(once) == 1 - assert len(twice) == 1 +def _edge(graph, source, target, relation=None): + return next( + edge for edge in graph.edges + if edge.source == source + and edge.target == target + and (relation is None or edge.attributes.get("relation") == relation) + ) -def test_dedupe_nodes_collapses_by_id_last_wins(): - # #1327: a shared module anchor is emitted once per importing file; the - # --no-cluster raw writer must collapse same-id node dicts (#1317). - nodes = [ - {"id": "foundation", "label": "Foundation", "type": "module", "source_file": "A.swift"}, - {"id": "akit", "label": "AKit", "file_type": "code"}, - {"id": "foundation", "label": "Foundation", "type": "module", "source_file": "B.swift"}, +def test_dedupe_records_are_deterministic(): + nodes = [{"id": "a", "label": "old"}, {"id": "b"}, {"id": "a", "label": "new"}] + assert dedupe_nodes(nodes) == [{"id": "a", "label": "new"}, {"id": "b"}] + edges = [ + {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, + {"source": "a", "target": "b", "relation": "calls", "source_location": "L2"}, + {"source": "a", "target": "b", "relation": "imports"}, ] - out = dedupe_nodes(nodes) - ids = [n["id"] for n in out] - assert ids == ["foundation", "akit"] # first-appearance order - # last writer wins on attributes - assert next(n for n in out if n["id"] == "foundation")["source_file"] == "B.swift" + assert dedupe_edges(edges) == [edges[0], edges[2]] -def load_extraction(): - return json.loads((FIXTURES / "extraction.json").read_text()) -def test_build_from_json_node_count(): - G = build_from_json(load_extraction()) - assert G.number_of_nodes() == 4 - -def test_build_from_json_edge_count(): - G = build_from_json(load_extraction()) - assert G.number_of_edges() == 4 - -def test_null_weight_edge_builds_and_clusters(tmp_path): - """#1960: an explicit ``"weight": null`` (JSON null -> None) used to survive - ``.get("weight", 1.0)`` and crash Louvain/Leiden modularity with a TypeError. - It must now coerce to the 1.0 default, build, and cluster without raising.""" - from graphify.cluster import cluster - extraction = { +def test_build_normalizes_attributes_weights_and_direction(): + graph = build_from_extraction({ "nodes": [ - {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, - {"id": "c", "label": "C", "file_type": "code", "source_file": "c.py"}, + {"id": "a", "label": "A", "source": "src\\a.py", "file_type": None, "_origin": "ast"}, + {"id": "b", "label": "B", "source_file": "src/b.py", "file_type": "code", "_origin": "ast"}, ], "edges": [ - {"source": "a", "target": "b", "relation": "references", "weight": None, + {"from": "a", "to": "b", "relation": "calls", "weight": None, "confidence_score": None}, - {"source": "b", "target": "c", "relation": "references", "weight": 2.5}, - ], - } - G = build_from_json(extraction) - assert G["a"]["b"]["weight"] == 1.0 # null coerced to default - assert G["a"]["b"]["confidence_score"] == 1.0 # null confidence_score too - assert G["b"]["c"]["weight"] == 2.5 # a valid weight is preserved - cluster(G) # must not raise (Louvain/Leiden modularity) - - -def test_malformed_weights_normalize(): - """Non-numeric / NaN / inf / negative weights fall back to 1.0 (the backends - reject them); a missing weight key is left absent (backends default it).""" - extraction = { - "nodes": [{"id": f"n{i}", "label": str(i), "file_type": "code", - "source_file": f"{i}.py"} for i in range(4)], - "edges": [ - {"source": "n0", "target": "n1", "relation": "references", "weight": "3.5"}, - {"source": "n1", "target": "n2", "relation": "references", "weight": float("nan")}, - {"source": "n2", "target": "n3", "relation": "references", "weight": -4}, ], - } - G = build_from_json(extraction) - assert G["n0"]["n1"]["weight"] == 3.5 # numeric string coerces - assert G["n1"]["n2"]["weight"] == 1.0 # NaN -> default - assert G["n2"]["n3"]["weight"] == 1.0 # negative -> default - - -def test_nodes_have_label(): - G = build_from_json(load_extraction()) - assert G.nodes["n_transformer"]["label"] == "Transformer" - -def test_edges_have_confidence(): - G = build_from_json(load_extraction()) - data = G.edges["n_attention", "n_concept_attn"] - assert data["confidence"] == "INFERRED" - -def test_ambiguous_edge_preserved(): - G = build_from_json(load_extraction()) - data = G.edges["n_layernorm", "n_concept_attn"] - assert data["confidence"] == "AMBIGUOUS" - -def test_legacy_node_source_canonicalized(): - """Legacy 'source' key on nodes is renamed to 'source_file' before graph build.""" - ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source": "a.py"}], - "edges": [], "input_tokens": 0, "output_tokens": 0} - G = build_from_json(ext) - assert "source_file" in G.nodes["n1"] - assert G.nodes["n1"]["source_file"] == "a.py" - assert "source" not in G.nodes["n1"] - - -def test_legacy_edge_from_to_canonicalized(): - """Legacy 'from'/'to' keys on edges are accepted alongside 'source'/'target'.""" - ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": "n2", "label": "B", "file_type": "code", "source_file": "b.py"}], - "edges": [{"from": "n1", "to": "n2", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "a.py", "weight": 1.0}], - "input_tokens": 0, "output_tokens": 0} - G = build_from_json(ext) - assert G.number_of_edges() == 1 - - -def test_source_file_backslash_normalized(): - """Windows backslash paths and POSIX paths for the same file must produce one node.""" - extraction = { - "nodes": [ - {"id": "n1", "label": "A", "file_type": "code", "source_file": "src\\middleware\\auth.py"}, - {"id": "n2", "label": "B", "file_type": "code", "source_file": "src/middleware/auth.py"}, - ], - "edges": [], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(extraction) - sources = {G.nodes[n]["source_file"] for n in G.nodes()} - assert sources == {"src/middleware/auth.py"} - - -def test_edge_missing_source_file_backfilled_from_node(): - """#1279: a semantic/LLM edge lacking source_file must inherit it from its - source node rather than reach graph.json with no file reference.""" - extraction = { - "nodes": [ - {"id": "n1", "label": "A", "file_type": "concept", "source_file": "docs/a.md"}, - {"id": "n2", "label": "B", "file_type": "concept", "source_file": "docs/b.md"}, - ], - # No source_file on the edge (as LLM output sometimes omits it). - "edges": [{"source": "n1", "target": "n2", "relation": "relates_to", "confidence": "INFERRED"}], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(extraction) - sf = edge_data(G, "n1", "n2").get("source_file") - assert sf == "docs/a.md" # backfilled from the source node - - -def test_build_merges_multiple_extractions(): - ext1 = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}], - "edges": [], "input_tokens": 0, "output_tokens": 0} - ext2 = {"nodes": [{"id": "n2", "label": "B", "file_type": "document", "source_file": "b.md"}], - "edges": [{"source": "n1", "target": "n2", "relation": "references", - "confidence": "INFERRED", "source_file": "b.md", "weight": 1.0}], - "input_tokens": 0, "output_tokens": 0} - G = build([ext1, ext2]) - assert G.number_of_nodes() == 2 - assert G.number_of_edges() == 1 - - -def test_none_file_type_defaults_to_concept(capsys): - """Legacy nodes with file_type=None (e.g. preserved from older graph.json - by `_rebuild_code`) must not trigger 'invalid file_type None' warnings (#660).""" - ext = { - "nodes": [ - {"id": "n1", "label": "Stub", "file_type": None, "source_file": "a.py"}, - {"id": "n2", "label": "Real", "file_type": "code", "source_file": "b.py"}, - ], - "edges": [], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(ext) - err = capsys.readouterr().err - assert "invalid file_type" not in err - # The legacy node still exists in the graph and has been canonicalized - assert G.nodes["n1"]["file_type"] == "concept" - assert G.nodes["n2"]["file_type"] == "code" - - -def test_missing_file_type_defaults_to_concept(capsys): - """Nodes missing file_type entirely should also be canonicalized to 'concept'.""" - ext = { - "nodes": [ - {"id": "n1", "label": "Bare", "source_file": "a.py"}, - ], - "edges": [], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(ext) - err = capsys.readouterr().err - assert "invalid file_type" not in err - assert "missing required field 'file_type'" not in err - assert G.nodes["n1"]["file_type"] == "concept" - - -def test_real_invalid_file_type_coerced_to_concept(): - """Unknown file_type values are coerced through the synonym mapper, falling - back to 'concept' for anything that isn't a known LLM synonym (#840).""" - ext = { - "nodes": [ - {"id": "n1", "label": "Bad", "file_type": "weird_type", "source_file": "a.py"}, - ], - "edges": [], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(ext) - assert G.nodes["n1"]["file_type"] == "concept" - - -def test_file_type_synonym_mapping(): - """Known invalid file_type values map to their canonical equivalents.""" - ext = { - "nodes": [ - {"id": "n1", "label": "MD", "file_type": "markdown", "source_file": "a.md"}, - {"id": "n2", "label": "Tool", "file_type": "tool", "source_file": "b.py"}, - {"id": "n3", "label": "Pat", "file_type": "pattern", "source_file": "c.md"}, - ], - "edges": [], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(ext) - assert G.nodes["n1"]["file_type"] == "document" - assert G.nodes["n2"]["file_type"] == "code" - assert G.nodes["n3"]["file_type"] == "concept" - - -def test_ghost_merge_unique_located_node_still_merges(): - """#1145 ghost-merge: a semantic ghost collapses into the single AST node - sharing its (basename, label), and edges re-point to the AST node.""" - ext = { - "nodes": [ - {"id": "ast_render", "label": "render", "file_type": "code", - "source_file": "src/app/index.ts", "source_location": "L10", "_origin": "ast"}, - {"id": "ghost_render", "label": "render", "file_type": "code", - "source_file": "src/app/index.ts"}, - {"id": "caller", "label": "main", "file_type": "code", - "source_file": "src/main.ts", "source_location": "L1", "_origin": "ast"}, - ], - "edges": [{"source": "caller", "target": "ghost_render", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "src/main.ts", "weight": 1.0}], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(ext) - assert "ghost_render" not in G.nodes() - assert G.has_edge("caller", "ast_render") - - -def test_ghost_merge_skipped_on_basename_collision(): - """#1257: when two files with the same basename both define a symbol with the - same label, the (basename, label) key is ambiguous and the semantic ghost - must not be merged into an arbitrary one of them.""" - ext = { - "nodes": [ - {"id": "a_render", "label": "render", "file_type": "code", - "source_file": "src/a/index.ts", "source_location": "L10", "_origin": "ast"}, - {"id": "b_render", "label": "render", "file_type": "code", - "source_file": "src/b/index.ts", "source_location": "L20", "_origin": "ast"}, - {"id": "ghost_render", "label": "render", "file_type": "code", - "source_file": "src/a/index.ts"}, - {"id": "caller", "label": "main", "file_type": "code", - "source_file": "src/main.ts", "source_location": "L1", "_origin": "ast"}, - ], - "edges": [{"source": "caller", "target": "ghost_render", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "src/main.ts", "weight": 1.0}], - "input_tokens": 0, "output_tokens": 0, - } - G = build_from_json(ext) - # The ghost survives: merging it into either a_render or b_render would - # pick an arbitrary winner (set iteration order over node_set). - assert "ghost_render" in G.nodes() - assert G.number_of_nodes() == 4 - assert G.has_edge("caller", "ghost_render") - assert not G.has_edge("caller", "a_render") - assert not G.has_edge("caller", "b_render") - - -def test_ghost_merge_non_ast_different_files_both_survive(): - """#1753: two NON-AST (semantic) nodes sharing (basename, label) but from - DIFFERENT files are distinct concepts with no AST canonical twin. They must - not be merged into an arbitrary survivor (which flipped run-to-run with the - hash seed); both survive, mirroring the AST/AST guard (#1257).""" - ext = { - "nodes": [ - {"id": "dir_a_update_build_merge", "label": "build_merge() function", - "file_type": "concept", "source_file": "dir_a/update.md", "source_location": "L10"}, - {"id": "dir_b_update_build_merge", "label": "build_merge() function", - "file_type": "concept", "source_file": "dir_b/update.md", "source_location": "L12"}, - ], - "edges": [], - } - G = build_from_json(ext, directed=False) - assert sorted(G.nodes()) == ["dir_a_update_build_merge", "dir_b_update_build_merge"] - - -def test_ghost_merge_non_ast_same_file_still_merges(): - """A genuine duplicate — two non-AST nodes with the SAME source_file and - label — is a real ghost and still collapses to one node (deterministically), - so #1753's fix doesn't leave same-file LLM duplicates behind.""" - ext = { - "nodes": [ - {"id": "a_foo", "label": "Foo", "file_type": "concept", - "source_file": "x/doc.md", "source_location": "L1"}, - {"id": "b_foo", "label": "Foo", "file_type": "concept", - "source_file": "x/doc.md", "source_location": "L2"}, - ], - "edges": [], - } - G = build_from_json(ext, directed=False) - assert G.number_of_nodes() == 1 - - -def test_build_merge_preserves_call_edge_direction(tmp_path): - """Regression for #760. - - When the callee is defined before the caller in source, NetworkX's - undirected Graph stores edges in node-insertion order. Going through - node_link_graph() + edges() during build_merge previously flipped the - `calls` edge so that on the next save source/target were swapped. - - build_merge must read the saved JSON's source/target verbatim instead - of round-tripping through NetworkX. - """ - from graphify.extract import extract_js - from graphify.export import to_json - - # Callee `b` is defined before caller `a` so node insertion order - # is b, a. An undirected Graph then yields the edge as (b, a) on - # iteration, which is the wrong direction for `calls` (a calls b). - src = "function b() {}\nfunction a() { b(); }\n" - src_file = tmp_path / "x.js" - src_file.write_text(src) - - extraction = extract_js(src_file) - assert "error" not in extraction - - # Locate the `calls` edge in the raw extraction so we know the truth. - call_edges = [e for e in extraction["edges"] if e["relation"] == "calls"] - assert len(call_edges) == 1, "expected exactly one calls edge from the snippet" - truth_src = call_edges[0]["source"] - truth_tgt = call_edges[0]["target"] - - nodes_by_id = {n["id"]: n for n in extraction["nodes"]} - assert nodes_by_id[truth_src]["label"].startswith("a") - assert nodes_by_id[truth_tgt]["label"].startswith("b") - - # First build + save. - G1 = build([extraction], dedup=False) - graph_path = tmp_path / "graph.json" - communities: dict = {} - assert to_json(G1, communities, str(graph_path), force=True) - - # Verify direction is correct in the freshly written JSON. - saved = json.loads(graph_path.read_text()) - saved_calls = [e for e in saved.get("links", saved.get("edges", [])) - if e.get("relation") == "calls"] - assert len(saved_calls) == 1 - assert saved_calls[0]["source"] == truth_src - assert saved_calls[0]["target"] == truth_tgt - - # Now simulate `--update` with no new chunks — load + re-save. - G2 = build_merge([], graph_path, dedup=False) - assert to_json(G2, communities, str(graph_path), force=True) - - # The calls edge must still go a -> b, not b -> a. - reloaded = json.loads(graph_path.read_text()) - reloaded_calls = [e for e in reloaded.get("links", reloaded.get("edges", [])) - if e.get("relation") == "calls"] - assert len(reloaded_calls) == 1 - assert reloaded_calls[0]["source"] == truth_src, ( - f"calls edge source flipped after build_merge round-trip: " - f"expected {truth_src} (a), got {reloaded_calls[0]['source']}" - ) - assert reloaded_calls[0]["target"] == truth_tgt, ( - f"calls edge target flipped after build_merge round-trip: " - f"expected {truth_tgt} (b), got {reloaded_calls[0]['target']}" - ) - - -def test_build_from_json_preserves_first_direction_on_bidirectional_pair(tmp_path): - """Regression for #1061. - - When an extraction emits two `calls` edges between the same pair in - opposite directions (mutual recursion, callbacks, event handlers, etc.), - nx.Graph collapses them into a single undirected edge. The deterministic - edge sort introduced in #1010 ordered edges by (source, target, relation), - so the lexicographically-later direction always wrote second and clobbered - the first edge's _src/_tgt — the surviving edge then exported with caller - and callee systematically swapped on every collision. - - build_from_json must keep the first-seen direction for the surviving edge - instead of letting the second add_edge overwrite _src/_tgt. - """ - from graphify.export import to_json - - # Lexicographic order of (src, tgt, rel) puts `a` < `z` first, so the sort - # processes `a -> z` BEFORE `z -> a`. Without the fix, the second write - # overwrites _src/_tgt and the exported edge becomes z -> a. With the fix, - # the first-seen `a -> z` direction is preserved. - extraction = { - "nodes": [ - {"id": "a_handler", "label": "a", "file_type": "code", "source_file": "a.ts"}, - {"id": "z_emitter", "label": "z", "file_type": "code", "source_file": "z.ts"}, - ], - "edges": [ - {"source": "a_handler", "target": "z_emitter", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "a.ts"}, - {"source": "z_emitter", "target": "a_handler", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "z.ts"}, - ], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(extraction) - # Only one undirected edge between the pair survives, but its stored - # direction must be the first-seen one (a_handler -> z_emitter), not the - # lexicographically-later one (z_emitter -> a_handler). - assert G.number_of_edges() == 1 - data = edge_data(G, "a_handler", "z_emitter") - assert data["_src"] == "a_handler" - assert data["_tgt"] == "z_emitter" - - graph_path = tmp_path / "graph.json" - assert to_json(G, {}, str(graph_path), force=True) - saved = json.loads(graph_path.read_text()) - saved_calls = [e for e in saved.get("links", saved.get("edges", [])) - if e.get("relation") == "calls"] - assert len(saved_calls) == 1 - assert saved_calls[0]["source"] == "a_handler", ( - f"calls edge source flipped on bidirectional collision: " - f"expected a_handler, got {saved_calls[0]['source']}" - ) - assert saved_calls[0]["target"] == "z_emitter", ( - f"calls edge target flipped on bidirectional collision: " - f"expected z_emitter, got {saved_calls[0]['target']}" - ) - - -# Regression tests for #796 — edge_data / edge_datas helpers must tolerate -# MultiGraph and MultiDiGraph, which networkx's node_link_graph() produces -# whenever the loaded JSON has multigraph: true. Plain G.edges[u, v] crashes -# on those with `ValueError: not enough values to unpack (expected 3, got 2)`. - -def test_edge_data_simple_graph(): - G = nx.Graph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d["relation"] == "calls" - assert d["confidence"] == "EXTRACTED" - - -def test_edge_datas_simple_graph_returns_singleton_list(): - G = nx.Graph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - ds = edge_datas(G, "a", "b") - assert isinstance(ds, list) - assert len(ds) == 1 - assert ds[0]["relation"] == "calls" - - -def test_edge_data_multigraph_with_parallel_edges(): - G = nx.MultiGraph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - G.add_edge("a", "b", relation="references", confidence="INFERRED") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - # First parallel edge wins; should be one of the two attribute dicts above. - assert d.get("relation") in ("calls", "references") - - -def test_edge_datas_multigraph_returns_all_parallel_edges(): - G = nx.MultiGraph() - G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") - G.add_edge("a", "b", relation="references", confidence="INFERRED") - ds = edge_datas(G, "a", "b") - assert isinstance(ds, list) - assert len(ds) == 2 - relations = {e.get("relation") for e in ds} - assert relations == {"calls", "references"} - - -def test_edge_data_multidigraph(): - G = nx.MultiDiGraph() - G.add_edge("a", "b", relation="calls") - G.add_edge("a", "b", relation="imports") - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d.get("relation") in ("calls", "imports") - ds = edge_datas(G, "a", "b") - assert len(ds) == 2 - - -def test_edge_data_node_link_multigraph_roundtrip(): - """A node_link JSON with multigraph: true must load as MultiGraph and the - helpers must operate on it without raising the 3-tuple unpack ValueError.""" - data = { - "directed": False, + }, directed=True) + assert graph.kind == "digraph" + assert graph.node_count == 2 and graph.edge_count == 1 + assert _attrs(graph, "a")["source_file"] == "src/a.py" + assert _attrs(graph, "a")["file_type"] == "concept" + assert _edge(graph, "a", "b").attributes["weight"] == 1.0 + assert _edge(graph, "a", "b").attributes["confidence_score"] == 1.0 + + +def test_build_merges_last_node_attributes_and_backfills_edge_source(): + graph = build([ + {"nodes": [{"id": "a", "label": "Old", "source_file": "a.py"}], "edges": []}, + { + "nodes": [ + {"id": "a", "label": "New", "source_file": "a.py"}, + {"id": "b", "label": "B", "source_file": "b.py"}, + ], + "edges": [{"source": "a", "target": "b", "relation": "references"}], + }, + ], dedup=False) + assert _attrs(graph, "a")["label"] == "New" + assert _edge(graph, "a", "b").attributes["source_file"] == "a.py" + + +def test_parallel_relations_and_self_loop_survive(): + graph = build_from_extraction({ + "directed": True, "multigraph": True, - "graph": {}, - "nodes": [ - {"id": "a", "label": "A"}, - {"id": "b", "label": "B"}, - ], + "nodes": [{"id": "a"}, {"id": "b"}], "links": [ - {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "a", "target": "b", "relation": "references", "confidence": "INFERRED"}, + {"source": "a", "target": "b", "key": "call", "relation": "calls"}, + {"source": "a", "target": "b", "key": "use", "relation": "uses"}, + {"source": "a", "target": "a", "key": "self", "relation": "recursive"}, ], - } - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) - assert isinstance(G, nx.MultiGraph) - # Plain G.edges[u, v] would raise here; the helper must not. - d = edge_data(G, "a", "b") - assert isinstance(d, dict) - assert d.get("relation") in ("calls", "references") - ds = edge_datas(G, "a", "b") - assert len(ds) == 2 + }) + assert graph.kind == "multidigraph" + assert graph.edge_count == 3 + assert {edge.key for edge in graph.edges} == {"call", "use", "self"} -def test_build_from_json_relativizes_absolute_source_file(tmp_path): - """Semantic subagents emit absolute source_file paths; build_from_json must - relativize them to root so MCP traversal works correctly (#932).""" - root = tmp_path / "myproject" - root.mkdir() - abs_path = str(root / "docs" / "overview.md") - extraction = { - "nodes": [ - {"id": "overview_intro", "label": "Intro", "source_file": abs_path, "file_type": "document"}, - ], +def test_unclustered_build_preserves_parallel_and_external_edges(): + graph = build_unclustered_extraction({ + "nodes": [{"id": "a"}, {"id": "b"}], "edges": [ - {"source": "overview_intro", "target": "overview_intro", - "relation": "self", "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": abs_path}, - ], - } - G = build_from_json(extraction, root=root) - # The id-stem migration (#1504) re-keys the old short id to the full-path form. - sf = G.nodes["docs_overview_intro"]["source_file"] - assert not sf.startswith("/"), f"source_file still absolute: {sf}" - assert sf == "docs/overview.md" - - -def test_build_relativizes_absolute_source_file(tmp_path): - """build() passes root through to build_from_json (#932).""" - root = tmp_path / "proj" - root.mkdir() - abs_path = str(root / "src" / "main.py") - extraction = { - "nodes": [{"id": "main_fn", "label": "main", "source_file": abs_path, "file_type": "code"}], + {"source": "a", "target": "b", "relation": "calls"}, + {"source": "a", "target": "b", "relation": "imports"}, + {"source": "b", "target": "a", "relation": "calls"}, + {"source": "a", "target": "external", "relation": "imports"}, + {"source": "a", "target": "b", "relation": "calls"}, + ], + }) + + assert graph.kind == "multigraph" + assert {node.id for node in graph.nodes} == {"a", "b", "external"} + assert graph.edge_count == 4 + assert [edge.key for edge in graph.edges[:3]] == [0, 1, 2] + assert { + (edge.source, edge.target, edge.attributes["relation"]) + for edge in graph.edges + } == { + ("a", "b", "calls"), + ("a", "b", "imports"), + ("b", "a", "calls"), + ("a", "external", "imports"), + } + + +def test_hyperedges_prune_dangling_members(): + graph = build_from_extraction({ + "nodes": [{"id": "a"}, {"id": "b"}], "edges": [], - } - G = build([extraction], root=root) - # #1504 re-keys main_fn (old stem "main") to the full-path form "src_main". - sf = G.nodes["src_main_fn"]["source_file"] - assert sf == "src/main.py" - - -def test_build_from_json_ambiguous_old_stem_alias_stays_dangling(tmp_path): - """The #1504 old-stem alias (e.g. "ping.h" -> bare "ping") is meant to let a - stale-id edge from an un-re-extracted fragment still find its own file after - a rekey. But the old-stem form drops the extension and most of the path, so - two unrelated real files easily collapse onto the same bare alias (a C header - and a PHP script both named "ping", in different directories). A dangling - edge produced by an unrelated third file's own unscoped fallback id (e.g. the - C/C++ extractor's last-resort target for an #include it couldn't resolve to - a real path) must not silently ride that alias onto an arbitrary one of them - — it should stay dangling and get dropped, same as any other unresolvable - edge, rather than wire two unrelated files/languages together by accident.""" - root = tmp_path / "repo" - root.mkdir() - extraction = { - "nodes": [ - # Ids given in their canonical (post-extract.py, extension-stripped) - # form, matching what a real graphify update run would already have - # produced before build_from_json assembles the final graph. - {"id": "dev_monitoring_ping", "label": "ping.h", "file_type": "code", - "source_file": "Dev/monitoring/ping.h"}, - {"id": "www_pages_api_ping", "label": "ping.php", "file_type": "code", - "source_file": "www/pages/api/ping.php"}, - {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", - "source_file": "Dev/poker/server.cpp"}, - ], - "edges": [ - # The unscoped, deliberately-unresolved fallback edge a C/C++ #include - # resolver leaves behind when it can't find the header on disk. - {"source": "dev_poker_server", "target": "ping", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, - ], - } - G = build_from_json(extraction, root=root) - assert not G.has_edge("dev_poker_server", "dev_monitoring_ping") - assert not G.has_edge("dev_poker_server", "www_pages_api_ping") - - -def test_build_from_json_ambiguous_alias_detected_despite_header_impl_salting(tmp_path): - """A same-directory .h/.cpp pair collides on their shared pre-extension id - and gets salted apart into ids like "tools_aolserver_utility_h_..." — no - longer a clean new_stem prefix. The ambiguity check must still recognize - the salted header as a legitimate claimant for the bare old-stem alias (by - label, not id shape), so a real collision with an unrelated same-named PHP - file is still caught instead of the header silently dropping out of the - race and leaving the PHP file as the lone "unambiguous" winner (this - reproduced against the real depot: Tools/aolserver/utility.h and .cpp, - salted apart, let wwwapi.masque.com/pages/utility.php win the bare - "utility" alias uncontested).""" - root = tmp_path / "repo" - root.mkdir() - extraction = { - "nodes": [ - {"id": "tools_aolserver_utility_h_tools_aolserver_utility", "label": "utility.h", - "file_type": "code", "source_file": "Tools/aolserver/utility.h"}, - {"id": "tools_aolserver_utility_cpp_tools_aolserver_utility", "label": "utility.cpp", - "file_type": "code", "source_file": "Tools/aolserver/utility.cpp"}, - {"id": "wwwapi_masque_com_pages_utility", "label": "utility.php", - "file_type": "code", "source_file": "wwwapi.masque.com/pages/utility.php"}, - {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", - "source_file": "Dev/poker/server.cpp"}, - ], - "edges": [ - {"source": "dev_poker_server", "target": "utility", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, - ], - } - G = build_from_json(extraction, root=root) - assert not G.has_edge("dev_poker_server", "wwwapi_masque_com_pages_utility") - assert not G.has_edge("dev_poker_server", "tools_aolserver_utility_h_tools_aolserver_utility") - - -def test_build_from_json_unambiguous_old_stem_alias_still_resolves(tmp_path): - """Companion to the ambiguous case above: when exactly one real file claims - an old-stem alias, a dangling edge to that bare alias should still resolve - to it — the #1504 migration-compat behavior this index exists for.""" - root = tmp_path / "repo" - root.mkdir() - extraction = { - "nodes": [ - {"id": "dev_monitoring_utility", "label": "utility.h", "file_type": "code", - "source_file": "Dev/monitoring/utility.h"}, - {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", - "source_file": "Dev/poker/server.cpp"}, - ], - "edges": [ - {"source": "dev_poker_server", "target": "utility", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, - ], - } - G = build_from_json(extraction, root=root) - assert G.has_edge("dev_poker_server", "dev_monitoring_utility") - - -def test_build_from_json_relative_source_file_unchanged(tmp_path): - """Already-relative source_file paths must not be modified.""" - extraction = { - "nodes": [{"id": "foo_bar", "label": "bar", "source_file": "src/foo.py", "file_type": "code"}], - "edges": [], - } - G = build_from_json(extraction, root=tmp_path) - # source_file must be untouched; the id is re-keyed to the full-path form (#1504). - assert G.nodes["src_foo_bar"]["source_file"] == "src/foo.py" - - -def test_build_merge_prune_absolute_paths_match_relative_nodes(tmp_path): - """#1007: manifest stores absolute paths, graph nodes store relative paths. - prune_sources with absolute paths must still remove the right nodes and edges.""" - import networkx as nx - - root = tmp_path / "corpus" - root.mkdir() - graph_path = tmp_path / "graph.json" - - # Simulate a graph with relative source_file paths (as built normally) - chunk = {"nodes": [ - {"id": "n1", "label": "login", "file_type": "code", "source_file": "module_a/auth.py"}, - {"id": "n2", "label": "format_date", "file_type": "code", "source_file": "module_b/utils.py"}, - ], "edges": [ - {"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED", - "source_file": "module_b/utils.py", "weight": 1.0}, - ]} - G0 = build([chunk], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") - - # prune_sources from manifest — absolute paths (what detect_incremental emits) - deleted_abs = [str(root / "module_b" / "utils.py")] - G1 = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False, root=root) - - node_labels = {d["label"] for _, d in G1.nodes(data=True)} - assert "format_date" not in node_labels, "stale node from deleted file should be pruned" - assert "login" in node_labels, "unrelated node must survive" - # Edge from deleted file must also be gone - assert G1.number_of_edges() == 0, "edge from deleted source_file should be pruned" - - -def test_build_merge_prune_windows_backslash_paths(tmp_path): - """#1007: prune_sources with Windows-style backslash absolute paths must still match.""" - import networkx as nx - - root = tmp_path / "corpus" - root.mkdir() - graph_path = tmp_path / "graph.json" - - chunk = {"nodes": [ - {"id": "n1", "label": "parse_date", "file_type": "code", "source_file": "module_b/utils.py"}, - ], "edges": []} - G0 = build([chunk], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") - - # Simulate Windows manifest path with backslashes - win_path = str(root / "module_b" / "utils.py").replace("/", "\\") - G1 = build_merge([], graph_path, prune_sources=[win_path], dedup=False, root=root) - - node_labels = {d["label"] for _, d in G1.nodes(data=True)} - assert "parse_date" not in node_labels, "node should be pruned even with backslash path" - - -def test_build_merge_replaces_changed_file_stale_edges(tmp_path): - """Re-extracting a CHANGED file must REPLACE its prior nodes/edges, not - accumulate them. build_merge previously only grew the graph, so an edge that - disappeared from a file's new version survived forever (only exact-duplicate - edges collapsed). The new-chunk source_file may be an absolute win32 path - while the stored graph keeps relative posix — both forms must match.""" - import networkx as nx - - root = tmp_path / "corpus" - root.mkdir() - graph_path = tmp_path / "graph.json" - - # First build: changed.md contributed A, B and edge A->B; keep.md is unrelated. - chunk0 = {"nodes": [ - {"id": "A", "label": "A", "file_type": "document", "source_file": "changed.md"}, - {"id": "B", "label": "B", "file_type": "document", "source_file": "changed.md"}, - {"id": "K", "label": "K", "file_type": "document", "source_file": "keep.md"}, - ], "edges": [ - {"source": "A", "target": "B", "relation": "references", "confidence": "EXTRACTED", - "source_file": "changed.md", "weight": 1.0}, - {"source": "K", "target": "A", "relation": "references", "confidence": "EXTRACTED", - "source_file": "keep.md", "weight": 1.0}, - ]} - G0 = build([chunk0], dedup=False) - graph_path.write_text(json.dumps(nx.node_link_data(G0, edges="edges")), encoding="utf-8") - - # changed.md edited: re-extraction now yields A, C and edge A->C (B dropped). - # source_file arrives as an absolute win32-style path (as detect emits on Windows). - abs_changed = str(root / "changed.md").replace("/", "\\") - new_chunk = {"nodes": [ - {"id": "A", "label": "A", "file_type": "document", "source_file": abs_changed}, - {"id": "C", "label": "C", "file_type": "document", "source_file": abs_changed}, - ], "edges": [ - {"source": "A", "target": "C", "relation": "references", "confidence": "EXTRACTED", - "source_file": abs_changed, "weight": 1.0}, - ]} - G1 = build_merge([new_chunk], graph_path, dedup=False, root=root) - - labels = {d["label"] for _, d in G1.nodes(data=True)} - edges = {(u, v) for u, v in G1.edges()} - - # Stale contribution from the old version of changed.md is gone. - assert "B" not in labels, "stale node from changed file's old version must be dropped" - assert ("A", "B") not in edges and ("B", "A") not in edges, "stale edge must be dropped" - # Fresh contribution is present. - assert "C" in labels, "re-extracted node must be present" - assert ("A", "C") in edges, "re-extracted edge must be present" - # An unchanged file is untouched. - assert "K" in labels, "unchanged file's node must survive" - assert ("K", "A") in edges, "unchanged file's edge must survive" - - -def test_build_merge_root_collapses_convention_drift(tmp_path): - """Skill contract: the extraction subagent must emit source_file as the - verbatim path from FILE_LIST AND the caller must pass root= (the build root). - Then build_merge canonicalizes the new chunk to the same relative base as the - stored graph, so re-extraction REPLACES the prior node (incl. stale nodes for - that file) instead of accumulating a duplicate. Without root, a drifted - relative base (e.g. a bare basename from a different run) mismatches and the - graph duplicates. Engine is unchanged — this pins the prompt/root contract.""" - import networkx as nx - - root = tmp_path - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - - # Stored graph: nested project-relative convention + a STALE node for the same - # file that the re-extraction no longer emits. - stored = {"nodes": [ - {"id": "wiki_overview_overview", "label": "Overview", "file_type": "document", - "source_file": "docs/wiki/overview.md"}, - {"id": "wiki_overview_stale", "label": "Stale", "file_type": "document", - "source_file": "docs/wiki/overview.md"}, - ], "edges": []} - G0 = build([stored], dedup=False) - saved = json.dumps(nx.node_link_data(G0, edges="edges")) - graph_path.write_text(saved, encoding="utf-8") - - # BUG: --update drifted to a bare basename and no root was passed. Different - # base -> source_file replace misses -> stale + duplicate both survive. - drift = {"nodes": [ - {"id": "overview_overview", "label": "Overview", "file_type": "document", - "source_file": "overview.md"}, - ], "edges": []} - G_bug = build_merge([drift], graph_path, dedup=False) - assert G_bug.number_of_nodes() == 3, "mismatched base must NOT replace -> stale+dup remain" - - # FIX: subagent emits the verbatim path; caller passes root (the build root). - graph_path.write_text(saved, encoding="utf-8") - abs_overview = str(root / "docs" / "wiki" / "overview.md") - fixed = {"nodes": [ - {"id": "wiki_overview_overview", "label": "Overview", "file_type": "document", - "source_file": abs_overview}, - ], "edges": []} - G_ok = build_merge([fixed], graph_path, prune_sources=None, dedup=False, root=root) - assert G_ok.number_of_nodes() == 1, "verbatim path + root must collapse to one node" - # #1504 re-keys the author-chosen short ids to the canonical full-path stem. - assert "docs_wiki_overview_stale" not in G_ok, "stale node for the re-extracted file must be dropped" - assert G_ok.nodes["docs_wiki_overview_overview"]["source_file"] == "docs/wiki/overview.md", \ - "new chunk must be canonicalized to the stored relative base" - - -def test_build_merge_rejects_oversized_existing_graph(monkeypatch, tmp_path): - """#F4: build_merge must refuse to read an existing graph.json that - exceeds the size cap, rather than json.loads-ing it into memory.""" - import pytest - - graph_path = tmp_path / "graph.json" - graph_path.write_text(json.dumps({"nodes": [], "links": []}), encoding="utf-8") - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(ValueError, match="exceeds"): - build_merge([], graph_path, dedup=False) - - -def test_build_from_json_skips_non_hashable_node_id(): - # A malformed LLM extraction can emit a list-valued id; build_from_json must - # skip it (NetworkX add_node would otherwise raise unhashable type) and still - # build the graph from the well-formed nodes. - extraction = { - "nodes": [ - {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": ["x", "y"], "label": "B", "file_type": "code", "source_file": "b.py"}, - {"label": "C", "file_type": "code", "source_file": "c.py"}, # missing id - ], - "edges": [], - } - G = build_from_json(extraction) - assert set(G.nodes()) == {"a"} - - -def test_build_from_json_skips_edge_with_non_hashable_endpoint(): - # A list-valued edge endpoint must be skipped rather than crash the - # `not in node_set` membership test. The well-formed edge survives. - extraction = { - "nodes": [ - {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, - ], - "edges": [ - {"source": "a", "target": ["b", "c"], "relation": "calls", - "confidence": "INFERRED", "source_file": "a.py"}, - {"source": "a", "target": "b", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "a.py"}, - ], - } - G = build_from_json(extraction) - assert G.number_of_nodes() == 2 - assert G.number_of_edges() == 1 - assert G.has_edge("a", "b") - - -# ── #1504 migration: legacy-id detection + re-key source_file contract ────────── - -def test_graph_has_legacy_ids_detects_old_scheme(): - """The read-only-consumer nudge (query/serve) flags a pre-#1504 graph and - leaves a canonical one alone.""" - from graphify.build import graph_has_legacy_ids - old = [{"id": "api_readme", "source_file": "docs/v1/api/README.md", "type": "document", "source_location": "L1"}] - new = [{"id": "docs_v1_api_readme", "source_file": "docs/v1/api/README.md", "type": "document", "source_location": "L1"}] - assert graph_has_legacy_ids(old, root=".") is True - assert graph_has_legacy_ids(new, root=".") is False - # sourceless / top-level file nodes don't false-positive - assert graph_has_legacy_ids([{"id": "setup", "source_file": "setup.py", "source_location": "L1"}], root=".") is False - assert graph_has_legacy_ids([{"id": "x", "label": "y"}], root=".") is False - # package/dir-scoped SYMBOL ids (Go's _make_id(pkg_dir, name) -> "sub_thing") must - # NOT false-positive: not file-level (no L1), so ignored even though "sub_thing" - # coincides with the old file-stem form of pkg/sub/thing.go. - go_symbol = [{"id": "sub_thing", "source_file": "pkg/sub/thing.go", "type": "code", "source_location": "L3"}] - assert graph_has_legacy_ids(go_symbol, root=".") is False - - -def test_semantic_rekey_relative_vs_absolute_source_file(): - """Re-key contract: a relative source_file is migrated; an absolute one is left - untouched (it can't be relativized, so its on-disk path must not leak into IDs).""" - from graphify.build import _semantic_id_remap - rel = [{"id": "api_readme", "source_file": "docs/v1/api/README.md", "type": "document"}] - assert _semantic_id_remap(rel, ".") == {"api_readme": "docs_v1_api_readme"} - # absolute path with no resolvable root → skipped, not remapped to an abs-path id - ab = [{"id": "api_readme", "source_file": "/abs/docs/v1/api/README.md", "type": "document"}] - assert _semantic_id_remap(ab, None) == {} - - -def test_cross_language_imports_references_are_dropped(): - """#1749: an `imports`/`references` edge must not bind across a language - family. A Python `import time` that resolved by bare stem onto a `time.ts` - file node welds the two language halves together at a phantom edge; the spec - forbids this for `calls` and it is equally invalid here.""" - ext = { - "nodes": [ - {"id": "backend_worker_py", "label": "worker.py", "file_type": "code", - "source_file": "backend/worker.py", "source_location": "L1", "_origin": "ast"}, - {"id": "src_time_ts", "label": "time.ts", "file_type": "code", - "source_file": "src/time.ts", "source_location": "L1", "_origin": "ast"}, - {"id": "src_util_ts", "label": "util.ts", "file_type": "code", - "source_file": "src/util.ts", "source_location": "L1", "_origin": "ast"}, - ], - "edges": [ - # phantom: Python file importing a TS file (cross-language) - {"source": "backend_worker_py", "target": "src_time_ts", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "backend/worker.py", "weight": 1.0}, - # legit: TS importing TS (same family) must survive - {"source": "src_time_ts", "target": "src_util_ts", "relation": "imports", - "confidence": "EXTRACTED", "source_file": "src/time.ts", "weight": 1.0}, - ], - } - G = build_from_json(ext, directed=False) - assert not G.has_edge("backend_worker_py", "src_time_ts"), "cross-language import must be dropped" - assert G.has_edge("src_time_ts", "src_util_ts"), "same-family (TS->TS) import must survive" - - -def test_cross_family_reference_to_unknown_ext_is_kept(): - """The #1749 guard only drops when BOTH endpoints are known code languages, - so a reference from a config/manifest (unknown ext) to a code file is kept.""" - ext = { - "nodes": [ - {"id": "pkg_json", "label": "package.json", "file_type": "code", - "source_file": "package.json", "source_location": "L1", "_origin": "ast"}, - {"id": "src_app_ts", "label": "app.ts", "file_type": "code", - "source_file": "src/app.ts", "source_location": "L1", "_origin": "ast"}, - ], - "edges": [ - {"source": "pkg_json", "target": "src_app_ts", "relation": "references", - "confidence": "EXTRACTED", "source_file": "package.json", "weight": 1.0}, + "hyperedges": [ + {"id": "flow", "nodes": ["a", "missing", "b"], "source_file": "flow.md"}, + {"id": "gone", "nodes": ["missing"]}, ], - } - G = build_from_json(ext, directed=False) - assert G.has_edge("pkg_json", "src_app_ts"), "config->code reference (unknown ext) must be kept" + }) + assert graph.attributes["hyperedges"] == [ + {"id": "flow", "nodes": ["a", "b"], "source_file": "flow.md"} + ] -def test_markdown_doc_twin_merges_into_semantic_doc_node(): - """#1799: the markdown quick-scan's bare `` doc node and the semantic - `_doc` node for the same file must collapse to one node, with edges - consolidated — otherwise a document is two disconnected halves and traversals - dead-end on the wrong twin.""" - ext = { +def test_incremental_merge_replaces_changed_and_prunes_deleted(tmp_path): + store_path = tmp_path / "graph.helix" + initial = build_from_extraction({ "nodes": [ - {"id": "docs_readme_doc", "label": "README", "file_type": "document", - "source_file": "docs/readme.md", "source_location": "L1"}, - {"id": "docs_readme", "label": "readme.md", "file_type": "document", - "source_file": "docs/readme.md", "source_location": "L1"}, - {"id": "code_auth", "label": "auth", "file_type": "code", - "source_file": "auth.py", "source_location": "L1"}, - {"id": "docs_guide", "label": "guide.md", "file_type": "document", - "source_file": "docs/guide.md", "source_location": "L1"}, + {"id": "a", "label": "Old", "source_file": "a.py", "_origin": "ast"}, + {"id": "stale", "label": "Stale", "source_file": "a.py", "_origin": "ast"}, + {"id": "b", "label": "B", "source_file": "b.py", "_origin": "ast"}, ], "edges": [ - {"source": "docs_readme_doc", "target": "code_auth", "relation": "references", - "source_file": "docs/readme.md", "confidence": "INFERRED", "weight": 1.0}, - {"source": "docs_guide", "target": "docs_readme", "relation": "references", - "source_file": "docs/guide.md", "confidence": "EXTRACTED", "weight": 1.0}, - ], - } - G = build_from_json(ext, directed=False) - assert "docs_readme" not in G.nodes() # bare twin merged away - assert "docs_readme_doc" in G.nodes() # semantic node is canonical - assert G.has_edge("docs_guide", "docs_readme_doc") # quick-scan edge repointed - assert G.has_edge("docs_readme_doc", "code_auth") # semantic edge kept - - -def test_doc_twin_merge_does_not_touch_code_symbols(): - """#1799 guard: a code symbol `foo` and an unrelated `foo_doc` (not - file_type=document) must NOT merge, even sharing a source_file.""" - ext = { - "nodes": [ - {"id": "m_foo", "label": "foo", "file_type": "code", - "source_file": "m.py", "source_location": "L1"}, - {"id": "m_foo_doc", "label": "foo rationale", "file_type": "rationale", - "source_file": "m.py", "source_location": "L2"}, - ], - "edges": [], - } - G = build_from_json(ext, directed=False) - assert {"m_foo", "m_foo_doc"} <= set(G.nodes()) - - -def test_build_from_json_prunes_dangling_hyperedge_members(capsys): - """#1916: build_from_json used to copy hyperedges into G.graph["hyperedges"] - verbatim without validating members, so a dangling member reached graph.json - even from a live (non-cache) extraction. Members absent from the built node - set are pruned — matching how dangling pairwise edges are skipped — and a - hyperedge with no surviving member is dropped whole.""" - ext = { - "nodes": [ - {"id": "alpha", "label": "alpha", "file_type": "code", "source_file": "a.py"}, - {"id": "beta", "label": "beta", "file_type": "code", "source_file": "a.py"}, - ], - "edges": [], - "hyperedges": [ - {"id": "he_partial", "nodes": ["alpha", "beta", "ghost_member"], "source_file": "a.py"}, - {"id": "he_all_ghost", "nodes": ["ghost1", "ghost2"], "source_file": "a.py"}, - ], - } - G = build_from_json(ext) - hes = {h["id"]: h for h in G.graph.get("hyperedges", [])} - assert set(hes) == {"he_partial"}, "an all-dangling hyperedge must be dropped" - assert hes["he_partial"]["nodes"] == ["alpha", "beta"] - assert "he_all_ghost" in capsys.readouterr().err + {"source": "a", "target": "b", "relation": "calls", "source_file": "a.py", "_origin": "ast"}, + {"source": "stale", "target": "b", "relation": "calls", "source_file": "a.py", "_origin": "ast"}, + ], + "hyperedges": [{"id": "b-flow", "nodes": ["b"], "source_file": "b.py"}], + }, directed=True) + merged = build_merge([ + {"nodes": [{"id": "a", "label": "New", "source_file": "a.py", "_origin": "ast"}], "edges": []} + ], graph_path=store_path, base_graph=initial, prune_sources=["b.py"], directed=True, dedup=False) + assert {node.id for node in merged.nodes} == {"a"} + assert _attrs(merged, "a")["label"] == "New" + assert merged.edge_count == 0 + assert merged.attributes.get("hyperedges", []) == [] + + +def test_typed_identifiers_remain_distinct(): + graph = build_from_extraction({ + "nodes": [{"id": 1, "label": "integer"}, {"id": "1", "label": "string"}], + "edges": [{"source": 1, "target": "1", "relation": "links"}], + }) + assert {node.id for node in graph.nodes} == {1, "1"} diff --git a/tests/test_build_merge_hyperedges_and_prune.py b/tests/test_build_merge_hyperedges_and_prune.py index aae58ee9c..9863e19d2 100644 --- a/tests/test_build_merge_hyperedges_and_prune.py +++ b/tests/test_build_merge_hyperedges_and_prune.py @@ -1,209 +1,15 @@ -"""Incremental --update: hyperedge preservation (#1574) and root-less prune (#1571). - -build_merge backs `graphify --update`. Two regressions covered here: - -- #1574: it read only nodes+edges from the existing graph.json, never hyperedges, - so every incremental update collapsed the graph's hyperedge set down to just the - re-extracted files'. Now existing hyperedges are carried forward, with - re-extracted files' replaced (by source_file) and deleted files' pruned. -- #1571: when a caller omits `root` (the skill's --update runbook does), absolute - prune_sources never relativized to match the stored relative source_file keys, so - deleted files' nodes survived as ghosts. build_merge now infers a fallback root. -""" -from __future__ import annotations - -import json -import os -from pathlib import Path - -import pytest - -from graphify.build import build_merge, _infer_merge_root - - -def _write_graph(graph_path: Path, nodes, edges, hyperedges) -> None: - """Write a graph.json in the shape to_json emits (top-level hyperedges).""" - graph_path.write_text( - json.dumps({"nodes": nodes, "edges": edges, "hyperedges": hyperedges}), - encoding="utf-8", - ) - - -def _he_ids(G) -> set[str]: - return {h["id"] for h in G.graph.get("hyperedges", [])} - - -# ── #1574: hyperedge preservation ───────────────────────────────────────────── - -def _seed_two_file_graph(tmp_path): - root = tmp_path / "corpus" - root.mkdir() - graph_path = tmp_path / "graph.json" - nodes = [ - {"id": "a1", "label": "a1", "file_type": "document", "source_file": "a.md"}, - {"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}, - ] - hyperedges = [ - {"id": "he_a", "label": "flow A", "source_file": "a.md", "nodes": ["a1"]}, - {"id": "he_b", "label": "flow B", "source_file": "b.md", "nodes": ["b1"]}, - {"id": "he_global", "label": "cross-file flow", "nodes": ["a1", "b1"]}, # no source_file - ] - _write_graph(graph_path, nodes, [], hyperedges) - return root, graph_path - - -def test_update_preserves_hyperedges_of_unchanged_files(tmp_path): - root, graph_path = _seed_two_file_graph(tmp_path) - # Re-extract only b.md, with a fresh hyperedge for it. - new_chunk = { - "nodes": [{"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}], - "edges": [], - "hyperedges": [{"id": "he_b_v2", "label": "flow B v2", "source_file": "b.md", "nodes": ["b1"]}], - } - G = build_merge([new_chunk], graph_path, dedup=False, root=root) - ids = _he_ids(G) - assert "he_a" in ids # unchanged file's hyperedge preserved (the bug) - assert "he_global" in ids # source_file-less hyperedge preserved - assert "he_b_v2" in ids # re-extracted file's new hyperedge present - assert "he_b" not in ids # re-extracted file's OLD hyperedge replaced - - -def test_update_without_root_still_preserves_hyperedges(tmp_path): - """The runbook omits root; the fallback root must not break preservation.""" - root, graph_path = _seed_two_file_graph(tmp_path) - new_chunk = { - "nodes": [{"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}], +from graphify.build import build_from_extraction, build_merge +def test_unchanged_hyperedges_are_carried_forward(tmp_path): + path = tmp_path / "graph.helix" + initial = build_from_extraction({ + "nodes": [{"id": "a", "source_file": "a.py"}, {"id": "b", "source_file": "b.py"}], "edges": [], - "hyperedges": [{"id": "he_b_v2", "source_file": "b.md", "nodes": ["b1"]}], - } - G = build_merge([new_chunk], graph_path, dedup=False) # no root - ids = _he_ids(G) - assert {"he_a", "he_global", "he_b_v2"} <= ids - assert "he_b" not in ids - - -def test_deleted_file_hyperedges_are_pruned(tmp_path): - root, graph_path = _seed_two_file_graph(tmp_path) - deleted_abs = [str(root / "a.md")] - G = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False, root=root) - ids = _he_ids(G) - assert "he_a" not in ids # deleted file's hyperedge pruned - assert "he_b" in ids # untouched file's hyperedge kept - assert "he_global" in ids # global hyperedge kept - # and its node is gone too - assert "a1" not in set(G.nodes) - - -# ── #1571: root-less prune (absolute deleted paths vs relative node keys) ────── - -def test_prune_without_root_removes_ghost_nodes_via_grandparent_fallback(tmp_path): - root = tmp_path / "corpus" - (root / "graphify-out").mkdir(parents=True) - graph_path = root / "graphify-out" / "graph.json" - nodes = [ - {"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}, - {"id": "k1", "label": "keep", "file_type": "document", "source_file": "KEEP.md"}, - ] - _write_graph(graph_path, nodes, [], []) - # Runbook-style call: absolute prune path, NO root passed. - deleted_abs = [str(root / "HANDOFF.md")] - G = build_merge([], graph_path, prune_sources=deleted_abs, dedup=False) - labels = {d["label"] for _, d in G.nodes(data=True)} - assert "handoff" not in labels, "deleted file's ghost node must be pruned without root" - assert "keep" in labels - - -def test_prune_without_root_uses_graphify_root_marker(tmp_path): - # graph.json not under a /graphify-out layout, so grandparent wouldn't - # help — the committed .graphify_root marker must be honored instead. - out = tmp_path / "out" - out.mkdir() - graph_path = out / "graph.json" - real_root = tmp_path / "elsewhere" / "repo" - real_root.mkdir(parents=True) - (out / ".graphify_root").write_text(str(real_root), encoding="utf-8") - nodes = [{"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}] - _write_graph(graph_path, nodes, [], []) - assert _infer_merge_root(graph_path) == str(real_root.resolve()) - G = build_merge([], graph_path, prune_sources=[str(real_root / "HANDOFF.md")], dedup=False) - assert "handoff" not in {d["label"] for _, d in G.nodes(data=True)} - - -@pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics") -def test_prune_matches_across_symlinked_root(tmp_path): - """A symlinked scan root (macOS /var -> /private/var, symlinked home/worktree) - makes the absolute prune path and the resolved root differ by prefix. The prune - must still match — lexical relative_to fails, so normalization resolves both - sides. Regression for the edge case a canonical-tmp unit test can't reach.""" - real = tmp_path / "real" - (real / "graphify-out").mkdir(parents=True) - link = tmp_path / "link" - os.symlink(real, link) - graph_path = real / "graphify-out" / "graph.json" - _write_graph(graph_path, [ - {"id": "h1", "label": "handoff", "file_type": "document", "source_file": "HANDOFF.md"}, - {"id": "k1", "label": "keep", "file_type": "document", "source_file": "KEEP.md"}, - ], [], []) - # prune path addressed via the SYMLINK, root resolved to the real dir - G = build_merge([], graph_path=graph_path, - prune_sources=[str(link / "HANDOFF.md")], root=str(real), dedup=False) - labels = {d["label"] for _, d in G.nodes(data=True)} - assert "handoff" not in labels and "keep" in labels - - -def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): - """#1796: a file present in BOTH new_chunks (re-extracted) and prune_sources - must be REPLACED, not deleted. The old edit-workflow passed the changed file - in prune_sources; combined with dedup keeping a same-label node, that used to - silently delete the freshly re-extracted concept. Replace wins over delete.""" - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - _write_graph( - graph_path, - nodes=[ - {"id": "foo_widget_cache", "label": "Widget Cache Design", - "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, - {"id": "bar_other", "label": "Other", - "file_type": "concept", "source_file": "docs/bar.md", "source_location": "L1"}, - ], - edges=[], - hyperedges=[], - ) - # foo.md edited: same-label node re-extracted (new content/line) - new_chunk = {"nodes": [ - {"id": "foo_widget_cache", "label": "Widget Cache Design", - "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} - ], "edges": []} - - G = build_merge([new_chunk], graph_path=str(graph_path), - prune_sources=["docs/foo.md"], root=str(tmp_path)) - labels = {G.nodes[n].get("label") for n in G.nodes()} - assert "Widget Cache Design" in labels, "re-extracted node was wrongly pruned" - - -def test_genuine_deletion_still_prunes(tmp_path): - """#1796 guard must not break real deletions: a file in prune_sources but NOT - in new_chunks is still removed.""" - graph_path = tmp_path / "graphify-out" / "graph.json" - graph_path.parent.mkdir(parents=True) - _write_graph( - graph_path, - nodes=[ - {"id": "foo_widget_cache", "label": "Widget Cache Design", - "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, - {"id": "bar_other", "label": "Other", - "file_type": "concept", "source_file": "docs/bar.md", "source_location": "L1"}, - ], - edges=[], - hyperedges=[], + "hyperedges": [{"id": "flow", "nodes": ["b"], "source_file": "b.py"}], + }) + merged = build_merge( + [{"nodes": [{"id": "a", "source_file": "a.py"}], "edges": []}], + graph_path=path, + base_graph=initial, + dedup=False, ) - new_chunk = {"nodes": [ - {"id": "foo_widget_cache", "label": "Widget Cache Design", - "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} - ], "edges": []} - # bar.md genuinely deleted (not re-extracted) - G = build_merge([new_chunk], graph_path=str(graph_path), - prune_sources=["docs/bar.md"], root=str(tmp_path)) - labels = {G.nodes[n].get("label") for n in G.nodes()} - assert "Other" not in labels, "genuinely deleted file's node should be pruned" - assert "Widget Cache Design" in labels + assert merged.attributes["hyperedges"][0]["id"] == "flow" diff --git a/tests/test_cache.py b/tests/test_cache.py index 993c5a0d2..c2f32055d 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,1136 +1,119 @@ -"""Tests for graphify/cache.py.""" -import pytest -from pathlib import Path -from graphify.cache import file_hash, cache_dir, load_cached, save_cached, cached_files, clear_cache, _body_content - - -@pytest.fixture -def tmp_file(tmp_path): - f = tmp_path / "sample.txt" - f.write_text("hello world") - return f - - -@pytest.fixture -def cache_root(tmp_path): - return tmp_path - - -def test_file_hash_consistent(tmp_file): - """Same file gives same hash on repeated calls.""" - h1 = file_hash(tmp_file) - h2 = file_hash(tmp_file) - assert h1 == h2 - assert isinstance(h1, str) - assert len(h1) == 64 # SHA256 hex digest length - - -def test_file_hash_changes(tmp_path): - """Different file contents give different hashes.""" - f1 = tmp_path / "a.txt" - f2 = tmp_path / "b.txt" - f1.write_text("content one") - f2.write_text("content two") - assert file_hash(f1) != file_hash(f2) - - -def test_cache_roundtrip(tmp_file, cache_root): - """Save then load returns the same result dict.""" - result = {"nodes": [{"id": "n1", "label": "Node1"}], "edges": []} - save_cached(tmp_file, result, root=cache_root) - loaded = load_cached(tmp_file, root=cache_root) - assert loaded == result - - -def test_cache_miss_on_change(tmp_file, cache_root): - """After file content changes, load_cached returns None.""" - result = {"nodes": [], "edges": [{"source": "a", "target": "b"}]} - save_cached(tmp_file, result, root=cache_root) - # Modify the file - tmp_file.write_text("completely different content") - assert load_cached(tmp_file, root=cache_root) is None - - -def test_cached_files(tmp_path, cache_root): - """cached_files returns the set of cached hashes.""" - f1 = tmp_path / "file1.py" - f2 = tmp_path / "file2.py" - f1.write_text("alpha") - f2.write_text("beta") - - save_cached(f1, {"nodes": [], "edges": []}, root=cache_root) - save_cached(f2, {"nodes": [], "edges": []}, root=cache_root) - - hashes = cached_files(cache_root) - assert file_hash(f1, cache_root) in hashes - assert file_hash(f2, cache_root) in hashes - - -def test_clear_cache(tmp_file, cache_root): - """clear_cache removes all .json files from graphify-out/cache/ (all subdirs).""" - save_cached(tmp_file, {"nodes": [], "edges": []}, root=cache_root) - # Since v0.5.3 entries go into cache/ast/, not the flat cache/ dir - cache_base = cache_root / "graphify-out" / "cache" - assert len(list(cache_base.rglob("*.json"))) > 0 - clear_cache(cache_root) - assert len(list(cache_base.rglob("*.json"))) == 0 - - -def test_md_frontmatter_only_change_same_hash(tmp_path): - """Changing only frontmatter fields in a .md file does not change the hash.""" - f = tmp_path / "doc.md" - f.write_text("---\nreviewed: 2026-01-01\n---\n\n# Title\n\nBody text.") - h1 = file_hash(f) - f.write_text("---\nreviewed: 2026-04-09\n---\n\n# Title\n\nBody text.") - h2 = file_hash(f) - assert h1 == h2 - - -def test_md_body_change_different_hash(tmp_path): - """Changing the body of a .md file produces a different hash.""" - f = tmp_path / "doc.md" - f.write_text("---\nreviewed: 2026-01-01\n---\n\n# Title\n\nOriginal body.") - h1 = file_hash(f) - f.write_text("---\nreviewed: 2026-01-01\n---\n\n# Title\n\nChanged body.") - h2 = file_hash(f) - assert h1 != h2 - - -def test_md_no_frontmatter_hashed_normally(tmp_path): - """A .md file with no frontmatter is hashed by its full content.""" - f = tmp_path / "doc.md" - f.write_text("# Just a heading\n\nNo frontmatter here.") - h1 = file_hash(f) - f.write_text("# Just a heading\n\nDifferent content.") - h2 = file_hash(f) - assert h1 != h2 - - -def test_non_md_file_hashed_fully(tmp_path): - """Non-.md files are still hashed by their full content.""" - f = tmp_path / "script.py" - f.write_text("# comment\nx = 1") - h1 = file_hash(f) - f.write_text("# changed comment\nx = 1") - h2 = file_hash(f) - assert h1 != h2 - - -def test_body_content_strips_frontmatter(): - """_body_content correctly strips YAML frontmatter.""" - content = b"---\ntitle: Test\n---\n\nActual body." - assert _body_content(content) == b"\n\nActual body." - - -def test_body_content_no_frontmatter(): - """_body_content returns content unchanged when no frontmatter present.""" - content = b"No frontmatter here." - assert _body_content(content) == content - - -# --- #1259: frontmatter delimiters must be whole `---` lines ----------------- - -def test_body_content_hr_start_is_not_frontmatter(): - """A document opening with a ``----`` thematic break has no frontmatter; - a later ``---`` hr must not be mistaken for a close delimiter.""" - content = b"----\nIntro paragraph that must be hashed.\n\n---\nbody" - assert _body_content(content) == content - - -def test_body_content_dash_title_start_is_not_frontmatter(): - """``--- title`` on the first line is prose, not an open delimiter.""" - content = b"--- title\nIntro that must be hashed.\n\n---\nbody" - assert _body_content(content) == content - - -def test_body_content_dash_text_line_is_not_close_delimiter(): - """``--- text`` and ``----`` lines inside opened frontmatter are not the - close; without a proper close the content passes through unchanged.""" - content = b"---\ntitle: Test\nbody starts here\n--- not a delimiter\n----\nreal content" - assert _body_content(content) == content - - -def test_body_content_later_proper_close_skips_dash_text_lines(): - """A ``--- text`` line is skipped; the next whole ``---`` line closes.""" - content = b"---\ntitle: Test\nnote: --- inline\n---\nreal body" - assert _body_content(content) == b"\nreal body" - - -def test_body_content_well_formed_output_byte_identical(): - """For well-formed frontmatter the stripped body must stay byte-identical - to the historical substring implementation, so existing semantic-cache - hashes do not churn (re-extraction is billed LLM work).""" - cases = [ - # (input, output of the historical text.find("\n---")+4 algorithm) - (b"---\ntitle: Test\n---\n\nActual body.", b"\n\nActual body."), - (b"---\nreviewed: 2026-01-01\n---\n\n# Title\n\nBody text.", b"\n\n# Title\n\nBody text."), - # close delimiter with trailing whitespace keeps it in the body - (b"---\ntitle: Test\n--- \nbody", b" \nbody"), - # CRLF line endings - (b"---\r\ntitle: Test\r\n---\r\nbody", b"\r\nbody"), - # empty frontmatter block - (b"---\n---\nbody", b"\nbody"), - # close as the very last line, no trailing newline - (b"---\ntitle: Test\n---", b""), - ] - for content, expected in cases: - assert _body_content(content) == expected, content - - -def test_md_edit_above_hr_changes_hash(tmp_path): - """Editing content above a mid-document ``----`` break must change the - hash -- previously that region was silently excluded from hashing.""" - f = tmp_path / "doc.md" - f.write_text("----\nIntro paragraph.\n\n---\nbody") - h1 = file_hash(f) - f.write_text("----\nEdited intro paragraph.\n\n---\nbody") - h2 = file_hash(f) - assert h1 != h2 - - -# --- #777: portable cache source_file fields -------------------------------- -# ``save_cached`` relativizes ``source_file`` entries inside the cache file -# so a committed ``graphify-out/cache/`` is portable across machines and -# CI runners. ``load_cached`` re-absolutizes them so consumers (extract, -# merge into graph.json) see the same shape that fresh extraction emits. - -def test_save_cached_relativizes_source_file(tmp_path): - """The on-disk cache JSON contains forward-slash relative source_file - entries — no absolute prefix from the saving machine leaks in.""" - import json - from graphify.cache import save_cached, file_hash, cache_dir - - (tmp_path / "src").mkdir() - src = tmp_path / "src" / "foo.py" - src.write_text("def x(): pass\n") - abs_src = str(src.resolve()) - result = { - "nodes": [{"id": "n1", "label": "foo", "source_file": abs_src}], - "edges": [{"source": "n1", "target": "n1", "source_file": abs_src}], - } - save_cached(src, result, root=tmp_path, kind="ast") - - h = file_hash(src, tmp_path) - entry = cache_dir(tmp_path, "ast") / f"{h}.json" - on_disk = json.loads(entry.read_text(encoding="utf-8")) - node_sources = {n["source_file"] for n in on_disk["nodes"]} - edge_sources = {e["source_file"] for e in on_disk["edges"]} - assert node_sources == {"src/foo.py"}, ( - f"cache nodes must store relative source_file; got {node_sources}" - ) - assert edge_sources == {"src/foo.py"} - - -def test_load_cached_absolutizes_source_file(tmp_path): - """``load_cached`` returns the same absolute-path shape that a fresh - extraction produces, so consumers don't need to special-case cache - hits vs. fresh extraction.""" - from graphify.cache import save_cached, load_cached - - (tmp_path / "src").mkdir() - src = tmp_path / "src" / "foo.py" - src.write_text("def x(): pass\n") - abs_src = str(src.resolve()) - save_cached(src, { - "nodes": [{"id": "n1", "source_file": abs_src}], - "edges": [{"source": "n1", "target": "n1", "source_file": abs_src}], - }, root=tmp_path, kind="ast") - - loaded = load_cached(src, root=tmp_path, kind="ast") - assert loaded is not None - assert loaded["nodes"][0]["source_file"] == abs_src - assert loaded["edges"][0]["source_file"] == abs_src - - -def test_load_cached_passes_through_legacy_absolute_source_file(tmp_path): - """Cache entries written by an older graphify (with absolute source_file - inside) must still load correctly: the absolutize step is a no-op for - already-absolute values.""" - import json - from graphify.cache import load_cached, file_hash, cache_dir - - (tmp_path / "src").mkdir() - src = tmp_path / "src" / "foo.py" - src.write_text("pass\n") - abs_src = str(src.resolve()) - - # Hand-write a legacy-format cache entry (absolute source_file). - h = file_hash(src, tmp_path) - entry = cache_dir(tmp_path, "ast") / f"{h}.json" - entry.write_text(json.dumps({ - "nodes": [{"id": "n1", "source_file": abs_src}], - "edges": [], - })) - - loaded = load_cached(src, root=tmp_path, kind="ast") - assert loaded is not None - assert loaded["nodes"][0]["source_file"] == abs_src +"""Native generation-state extraction cache tests.""" +from pathlib import Path -def test_cache_portable_across_roots(tmp_path): - """End-to-end portability: a cache entry written at one root can be - consumed at a different absolute root because the file is content-hashed - AND its embedded source_file is stored relative.""" - import json - import shutil - from graphify.cache import save_cached, load_cached, file_hash, cache_dir - - repo_a = tmp_path / "repo_a" - repo_a.mkdir() - (repo_a / "src").mkdir() - src_a = repo_a / "src" / "foo.py" - src_a.write_text("def x(): pass\n") - save_cached(src_a, { - "nodes": [{"id": "n1", "source_file": str(src_a.resolve())}], - "edges": [], - }, root=repo_a, kind="ast") - - # Copy corpus + cache to a second location with a different absolute prefix. - repo_b = tmp_path / "repo_b" - shutil.copytree(repo_a, repo_b) - - src_b = repo_b / "src" / "foo.py" - loaded = load_cached(src_b, root=repo_b, kind="ast") - assert loaded is not None, ( - "cache must port across absolute prefixes (content hash + relative source_file)" - ) - # Source path re-anchored to the new root, not the old one. - assert loaded["nodes"][0]["source_file"] == str(src_b.resolve()) - assert not str(repo_a) in loaded["nodes"][0]["source_file"] - - -# --- AST cache versioning ---------------------------------------------------- -# AST cache entries are the output of graphify's own extractor code, so they -# are only valid for the graphify version that wrote them. Keying purely on -# file content meant extractor fixes shipped in a new release kept serving -# stale pre-fix results. The AST cache is therefore namespaced by package -# version; the semantic cache is NOT (invalidating it would re-bill LLM -# extraction for unchanged files). - -def test_ast_cache_invalidated_on_version_bump(tmp_path, monkeypatch): - """An AST entry written by version X must not be served after upgrading - to version Y — the file is unchanged but the extractor is not.""" - import graphify.cache as cache_mod - - f = tmp_path / "mod.py" - f.write_text("def f(): pass\n") - - monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False) - save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="ast") - assert load_cached(f, root=tmp_path, kind="ast") is not None - - monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False) - assert load_cached(f, root=tmp_path, kind="ast") is None, ( - "AST cache entry from a previous graphify version must not be served" - ) - - -def test_ast_cache_version_bump_cleans_stale_entries(tmp_path, monkeypatch): - """Upgrading removes AST entries left behind by previous versions so the - cache directory does not grow one full copy per release.""" - import graphify.cache as cache_mod - - f = tmp_path / "mod.py" - f.write_text("def f(): pass\n") - - monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False) - save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="ast") - old_dir = cache_dir(tmp_path, "ast") - assert any(old_dir.glob("*.json")) - - monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False) - monkeypatch.setattr(cache_mod, "_cleaned_ast_dirs", set(), raising=False) - cache_dir(tmp_path, "ast") - assert not old_dir.exists(), ( - "stale AST version directory must be removed on upgrade" - ) - - -def test_legacy_unversioned_ast_entries_not_served(tmp_path): - """Entries written by pre-versioning graphify (flat cache/ or unversioned - cache/ast/) are by definition from an older extractor and must not be - served — that staleness is exactly what version namespacing fixes.""" - import json - from graphify.cache import file_hash, _GRAPHIFY_OUT - - f = tmp_path / "mod.py" - f.write_text("def f(): pass\n") - h = file_hash(f, tmp_path) - payload = json.dumps({"nodes": [{"id": "stale"}], "edges": []}) - - # Unversioned cache/ast/{hash}.json (pre-versioning layout) - unversioned = tmp_path / _GRAPHIFY_OUT / "cache" / "ast" - unversioned.mkdir(parents=True) - (unversioned / f"{h}.json").write_text(payload) - # Legacy flat cache/{hash}.json (pre-0.5.3 layout) - (unversioned.parent / f"{h}.json").write_text(payload) - - assert load_cached(f, root=tmp_path, kind="ast") is None - - -def test_semantic_cache_survives_version_bump(tmp_path, monkeypatch): - """The semantic cache is deliberately not versioned: entries are produced - by the LLM from file contents, and re-extraction costs real money.""" - import graphify.cache as cache_mod - - f = tmp_path / "doc.md" - f.write_text("# Title\n\nBody.\n") - - monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.0", raising=False) - save_cached(f, {"nodes": [{"id": "n1"}], "edges": []}, root=tmp_path, kind="semantic") - semantic_dir = cache_dir(tmp_path, "semantic") - - monkeypatch.setattr(cache_mod, "_EXTRACTOR_VERSION", "0.8.1", raising=False) - monkeypatch.setattr(cache_mod, "_cleaned_ast_dirs", set(), raising=False) - cache_dir(tmp_path, "ast") # triggers stale-AST cleanup - assert load_cached(f, root=tmp_path, kind="semantic") is not None - assert any(semantic_dir.glob("*.json")), ( - "semantic entries must survive both the version bump and AST cleanup" - ) - - -def test_save_cached_in_root_symlink_keeps_symlink_name(tmp_path): - """``source_file`` for an in-root symlink must be stored under the - symlink's own name, not the resolved target. Lower-impact than the - manifest case (cache lookup is content-hashed, not key-matched), but - keeps the on-disk shape consistent with what callers passed in.""" - import json - from graphify.cache import save_cached, file_hash, cache_dir - - (tmp_path / "sub").mkdir() - target = tmp_path / "sub" / "target.py" - target.write_text("pass\n") - alias = tmp_path / "alias.py" - try: - alias.symlink_to(target) - except (OSError, NotImplementedError): - import pytest - pytest.skip("filesystem does not support symlinks") - - abs_alias = str(alias) # caller's view — the symlink path, unresolved - save_cached(alias, { - "nodes": [{"id": "n1", "source_file": abs_alias}], - "edges": [], - }, root=tmp_path, kind="ast") - - h = file_hash(alias, tmp_path) - entry = cache_dir(tmp_path, "ast") / f"{h}.json" - on_disk = json.loads(entry.read_text(encoding="utf-8")) - assert on_disk["nodes"][0]["source_file"] == "alias.py", ( - f"cache must store symlink name, not resolved target; got " - f"{on_disk['nodes'][0]['source_file']!r}" - ) - - -def test_semantic_prune_removes_orphan_entries(tmp_path): - """Changing a file's content leaves the old content-hash entry orphaned; - pruning against the new live hash removes the stale entry and keeps the - current one.""" - from graphify.cache import prune_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# A\n\nContent A.\n") - h_a = file_hash(f, tmp_path) - save_cached(f, {"nodes": [{"id": "a"}], "edges": []}, root=tmp_path, kind="semantic") - - f.write_text("# B\n\nContent B.\n") - h_b = file_hash(f, tmp_path) - save_cached(f, {"nodes": [{"id": "b"}], "edges": []}, root=tmp_path, kind="semantic") - - semantic_dir = cache_dir(tmp_path, "semantic") - assert (semantic_dir / f"{h_a}.json").exists() - assert (semantic_dir / f"{h_b}.json").exists() - - pruned = prune_semantic_cache(tmp_path, {h_b}) - assert pruned == 1 - assert not (semantic_dir / f"{h_a}.json").exists() - assert (semantic_dir / f"{h_b}.json").exists() - - -def test_semantic_prune_keeps_live_unchanged_entries(tmp_path): - """Pruning against the FULL live set must keep every live entry — guards - the trap of pruning against an incremental changed-subset, which would - delete all unchanged docs' valid entries.""" - from graphify.cache import prune_semantic_cache - - live_hashes = set() - for i in range(5): - f = tmp_path / f"doc{i}.md" - f.write_text(f"# Doc {i}\n\nBody {i}.\n") - save_cached(f, {"nodes": [{"id": str(i)}], "edges": []}, root=tmp_path, kind="semantic") - live_hashes.add(file_hash(f, tmp_path)) - - semantic_dir = cache_dir(tmp_path, "semantic") - assert len(list(semantic_dir.glob("*.json"))) == 5 - - pruned = prune_semantic_cache(tmp_path, live_hashes) - assert pruned == 0 - assert len(list(semantic_dir.glob("*.json"))) == 5 - - -def test_semantic_prune_handles_deleted_file(tmp_path): - """An entry for a file that no longer exists (dropped from the live set) is - pruned.""" - from graphify.cache import prune_semantic_cache - - f = tmp_path / "gone.md" - f.write_text("# Gone\n\nWill be deleted.\n") - h = file_hash(f, tmp_path) - save_cached(f, {"nodes": [{"id": "g"}], "edges": []}, root=tmp_path, kind="semantic") - semantic_dir = cache_dir(tmp_path, "semantic") - assert (semantic_dir / f"{h}.json").exists() - - f.unlink() - # Live set is empty: the file is gone, so its entry must be pruned. - pruned = prune_semantic_cache(tmp_path, set()) - assert pruned == 1 - assert not (semantic_dir / f"{h}.json").exists() - - -def test_semantic_prune_ignores_ast_and_tmp(tmp_path): - """Prune touches only cache/semantic/*.json: AST entries and atomic-write - *.tmp temporaries are left untouched.""" - from graphify.cache import prune_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n\nBody.\n") - # AST entry (different subtree) must survive. - save_cached(f, {"nodes": [{"id": "ast"}], "edges": []}, root=tmp_path, kind="ast") - ast_dir = cache_dir(tmp_path, "ast") - assert len(list(ast_dir.glob("*.json"))) == 1 - - # A semantic orphan .json (to be pruned) plus a .tmp temporary (to survive). - semantic_dir = cache_dir(tmp_path, "semantic") - (semantic_dir / "deadbeef.json").write_text('{"nodes": [], "edges": []}') - tmp_entry = semantic_dir / "deadbeef.tmp" - tmp_entry.write_text("partial") - - pruned = prune_semantic_cache(tmp_path, set()) - assert pruned == 1 - assert not (semantic_dir / "deadbeef.json").exists() - assert tmp_entry.exists(), "*.tmp temporaries must not be swept" - assert len(list(ast_dir.glob("*.json"))) == 1, "AST entries must not be touched" - - -def test_save_semantic_cache_overwrites_by_default(tmp_path): - """Default save_semantic_cache replaces a file's cached entry (the final, - authoritative write in the extract pipeline).""" - from graphify.cache import save_semantic_cache - f = tmp_path / "doc.md"; f.write_text("# Doc\n") - save_semantic_cache([{"id": "a", "source_file": "doc.md"}], [], root=tmp_path) - save_semantic_cache([{"id": "b", "source_file": "doc.md"}], [], root=tmp_path) - cached = load_cached(f, root=tmp_path, kind="semantic") - ids = {n["id"] for n in cached["nodes"]} - assert ids == {"b"}, "default must overwrite, not accumulate" - - -def test_save_semantic_cache_rejects_out_of_scope_source_file(tmp_path): - """#1757: an undispatched file must keep its complete cache entry when a - semantic result misattributes a node to it.""" - from graphify.cache import save_semantic_cache - - intended = tmp_path / "intended.md" - intended.write_text("# Intended\n") - protected = tmp_path / "protected.md" - protected.write_text("# Protected\n") - - save_semantic_cache( - [{"id": "original", "source_file": "protected.md"}], - [], +from graphify.cache import ( + _body_content, + cached_files, + check_semantic_cache, + clear_cache, + file_hash, + load_cached, + prune_semantic_cache, + save_cached, + save_semantic_cache, +) + + +def test_ast_cache_roundtrip_and_content_invalidation(tmp_path: Path) -> None: + source = tmp_path / "sample.py" + source.write_text("x = 1\n", encoding="utf-8") + cache: dict = {} + result = {"nodes": [{"id": "n", "source_file": str(source)}], "edges": []} + + save_cached(source, result, root=tmp_path, cache=cache) + + loaded = load_cached(source, root=tmp_path, cache=cache) + assert loaded == {"nodes": [{"id": "n", "source_file": str(source.resolve())}], "edges": []} + source.write_text("x = 2\n", encoding="utf-8") + assert load_cached(source, root=tmp_path, cache=cache) is None + + +def test_cache_state_contains_portable_paths_and_no_sidecar(tmp_path: Path) -> None: + source = tmp_path / "src" / "sample.py" + source.parent.mkdir() + source.write_text("pass\n", encoding="utf-8") + cache: dict = {} + + save_cached( + source, + {"nodes": [{"id": "n", "source_file": str(source.resolve())}], "edges": []}, root=tmp_path, - ) - - nodes = [ - {"id": "expected", "source_file": str(intended.resolve())}, - {"id": "stray", "source_file": "protected.md"}, - ] - edges = [ - {"source": "stray", "target": "expected", "source_file": "protected.md"}, - ] - hyperedges = [ - {"id": "stray_hyperedge", "nodes": ["stray"], "source_file": "protected.md"}, - ] - - with pytest.warns(RuntimeWarning, match="out-of-scope source_file 'protected.md'"): - saved = save_semantic_cache( - nodes, - edges, - hyperedges, - root=tmp_path, - allowed_source_files=["intended.md"], - ) - - assert saved == 1 - intended_cache = load_cached(intended, root=tmp_path, kind="semantic") - assert {node["id"] for node in intended_cache["nodes"]} == {"expected"} - - protected_cache = load_cached(protected, root=tmp_path, kind="semantic") - assert {node["id"] for node in protected_cache["nodes"]} == {"original"} - assert protected_cache["edges"] == [] - assert protected_cache["hyperedges"] == [] - - -# --- #1894: mode-namespaced semantic cache ----------------------------------- -# `extract --mode deep` produces richer results than standard extraction, so -# deep entries live in their own namespace (cache/semantic-deep/). mode=None -# must stay byte-identical to the historical behavior: older installed skill -# flows call check/save without the parameter and must be unaffected. - -def test_semantic_cache_deep_mode_roundtrip_under_deep_namespace(tmp_path): - """mode='deep' saves under cache/semantic-deep/ and reads back from it.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n\nBody.\n") - saved = save_semantic_cache( - [{"id": "deep_n", "source_file": "doc.md"}], [], root=tmp_path, mode="deep" - ) - assert saved == 1 - - deep_dir = tmp_path / "graphify-out" / "cache" / "semantic-deep" - h = file_hash(f, tmp_path) - assert (deep_dir / f"{h}.json").exists(), ( - "deep entry must land under cache/semantic-deep/" - ) - # And NOT in the plain namespace. - plain_dir = tmp_path / "graphify-out" / "cache" / "semantic" - assert not (plain_dir / f"{h}.json").exists() - - nodes, edges, hyper, uncached = check_semantic_cache( - [str(f)], root=tmp_path, mode="deep" - ) - assert [n["id"] for n in nodes] == ["deep_n"] - assert uncached == [] - - -def test_semantic_cache_deep_invisible_to_plain_reads_and_vice_versa(tmp_path): - """Deep entries must not satisfy mode=None reads (and plain entries must - not satisfy deep reads) — the namespaces are fully isolated.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - deep_doc = tmp_path / "deep.md" - deep_doc.write_text("# Deep\n") - plain_doc = tmp_path / "plain.md" - plain_doc.write_text("# Plain\n") - - save_semantic_cache([{"id": "d", "source_file": "deep.md"}], [], - root=tmp_path, mode="deep") - save_semantic_cache([{"id": "p", "source_file": "plain.md"}], [], - root=tmp_path) # mode omitted: historical call shape - - # Plain read: deep entry is a miss, plain entry is a hit. - nodes, _, _, uncached = check_semantic_cache( - [str(deep_doc), str(plain_doc)], root=tmp_path - ) - assert [n["id"] for n in nodes] == ["p"] - assert uncached == [str(deep_doc)] - - # Deep read: mirror image. - nodes, _, _, uncached = check_semantic_cache( - [str(deep_doc), str(plain_doc)], root=tmp_path, mode="deep" - ) - assert [n["id"] for n in nodes] == ["d"] - assert uncached == [str(plain_doc)] - - -def test_semantic_cache_mode_none_layout_unchanged(tmp_path): - """Omitting mode writes exactly the historical cache/semantic/ layout — - forward-compat for older installed callers that never pass mode.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], root=tmp_path) - h = file_hash(f, tmp_path) - assert (tmp_path / "graphify-out" / "cache" / "semantic" / f"{h}.json").exists() - assert not (tmp_path / "graphify-out" / "cache" / "semantic-deep").exists(), ( - "mode=None must never create the deep namespace" - ) - nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path) - assert [n["id"] for n in nodes] == ["n"] and uncached == [] - - -def test_clear_cache_removes_deep_namespace(tmp_path): - """clear_cache sweeps cache/semantic-deep/ alongside semantic/ and ast/.""" - from graphify.cache import save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "p", "source_file": "doc.md"}], [], root=tmp_path) - save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], - root=tmp_path, mode="deep") - base = tmp_path / "graphify-out" / "cache" - assert list((base / "semantic").glob("*.json")) - assert list((base / "semantic-deep").glob("*.json")) - - clear_cache(tmp_path) - assert not list(base.rglob("*.json")), ( - "clear_cache must remove entries in BOTH semantic namespaces" - ) - - -def test_cached_files_includes_deep_namespace(tmp_path): - """cached_files reports deep-namespace entries too.""" - from graphify.cache import save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], - root=tmp_path, mode="deep") - assert file_hash(f, tmp_path) in cached_files(tmp_path) - - -def test_semantic_prune_sweeps_both_namespaces_against_same_live_set(tmp_path): - """#1894 follow-up to #1527: prune must sweep cache/semantic/ AND - cache/semantic-deep/ against the SAME live-hash set (liveness is - content-based, mode-independent). Orphans go in both namespaces; live - entries survive in both.""" - from graphify.cache import prune_semantic_cache, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# A\n\nContent A.\n") - h_old = file_hash(f, tmp_path) - save_semantic_cache([{"id": "pa", "source_file": "doc.md"}], [], root=tmp_path) - save_semantic_cache([{"id": "da", "source_file": "doc.md"}], [], - root=tmp_path, mode="deep") - - f.write_text("# B\n\nContent B.\n") - h_live = file_hash(f, tmp_path) - save_semantic_cache([{"id": "pb", "source_file": "doc.md"}], [], root=tmp_path) - save_semantic_cache([{"id": "db", "source_file": "doc.md"}], [], - root=tmp_path, mode="deep") - - plain_dir = tmp_path / "graphify-out" / "cache" / "semantic" - deep_dir = tmp_path / "graphify-out" / "cache" / "semantic-deep" - for d in (plain_dir, deep_dir): - assert (d / f"{h_old}.json").exists() - assert (d / f"{h_live}.json").exists() - - pruned = prune_semantic_cache(tmp_path, {h_live}) - assert pruned == 2, "one orphan in EACH namespace must be pruned" - for d in (plain_dir, deep_dir): - assert not (d / f"{h_old}.json").exists(), f"orphan survived in {d.name}" - assert (d / f"{h_live}.json").exists(), f"live entry pruned from {d.name}" - - -def test_save_semantic_cache_merge_existing_unions(tmp_path): - """#1715: merge_existing=True unions with the prior entry so a file split - across chunks (checkpointed per chunk) keeps every slice.""" - from graphify.cache import save_semantic_cache - f = tmp_path / "big.md"; f.write_text("# Big\n") - # chunk 1 slice - save_semantic_cache([{"id": "a", "source_file": "big.md"}], - [{"source": "a", "target": "x", "source_file": "big.md"}], - root=tmp_path, merge_existing=True) - # chunk 2 slice for the same file - save_semantic_cache([{"id": "b", "source_file": "big.md"}], [], - root=tmp_path, merge_existing=True) - cached = load_cached(f, root=tmp_path, kind="semantic") - ids = {n["id"] for n in cached["nodes"]} - assert ids == {"a", "b"}, "merge_existing must union both chunk slices" - assert len(cached["edges"]) == 1 - - -def test_save_semantic_cache_drops_edges_to_out_of_scope_nodes(tmp_path): - """#1916: an edge in an ALLOWED file's group referencing a node grouped - under an out-of-scope REAL file used to be written verbatim, so on replay - (check_semantic_cache) it dangled forever — the #1895 merged-result filter - runs after this checkpoint write and is bypassed entirely on replay. The - written entry must carry no reference to the skipped id, while a - duplicate-attribution node (also defined in a written group) must not be - over-pruned.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - allowed = tmp_path / "allowed.md" - allowed.write_text("# Allowed\n") - outside = tmp_path / "outside.md" - outside.write_text("# Outside\n") - - nodes = [ - {"id": "kept", "source_file": "allowed.md"}, - {"id": "stray", "source_file": "outside.md"}, - # duplicate attribution: same id defined in a written AND a skipped group - {"id": "dup", "source_file": "allowed.md"}, - {"id": "dup", "source_file": "outside.md"}, - ] - edges = [ - {"source": "kept", "target": "stray", "source_file": "allowed.md"}, - {"source": "stray", "target": "kept", "source_file": "allowed.md"}, - {"source": "kept", "target": "dup", "source_file": "allowed.md"}, - ] - with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): - saved = save_semantic_cache( - nodes, edges, root=tmp_path, allowed_source_files=["allowed.md"] - ) - assert saved == 1 - - cached_nodes, cached_edges, _, uncached = check_semantic_cache( - [str(allowed)], root=tmp_path - ) - assert uncached == [] - assert {n["id"] for n in cached_nodes} == {"kept", "dup"} - pairs = [(e["source"], e["target"]) for e in cached_edges] - assert pairs == [("kept", "dup")], "edges touching the skipped id must be dropped" - - -def test_save_semantic_cache_drops_edges_to_ghost_file_nodes(tmp_path): - """#1916 (ghost variant): a node group whose source_file does not exist is - silently skipped by the write loop; edges in a written group referencing - its node ids must not survive into the cache.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - real = tmp_path / "real.md" - real.write_text("# Real\n") - - nodes = [ - {"id": "kept", "source_file": "real.md"}, - {"id": "phantom", "source_file": "ghost.md"}, # no such file on disk - ] - edges = [ - {"source": "kept", "target": "phantom", "source_file": "real.md"}, - {"source": "kept", "target": "kept", "relation": "self", "source_file": "real.md"}, - ] - saved = save_semantic_cache( - nodes, edges, root=tmp_path, allowed_source_files=["real.md"] - ) - assert saved == 1 - - cached_nodes, cached_edges, _, uncached = check_semantic_cache( - [str(real)], root=tmp_path - ) - assert uncached == [] - assert {n["id"] for n in cached_nodes} == {"kept"} - pairs = [(e["source"], e["target"]) for e in cached_edges] - assert pairs == [("kept", "kept")] - - -def test_save_semantic_cache_drops_hyperedges_touching_skipped_nodes(tmp_path): - """#1916: a hyperedge whose member list intersects the skipped ids is - dropped whole (mirroring the #1895 semantics), while hyperedges over - surviving nodes are kept.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - allowed = tmp_path / "allowed.md" - allowed.write_text("# Allowed\n") - outside = tmp_path / "outside.md" - outside.write_text("# Outside\n") - - nodes = [ - {"id": "kept", "source_file": "allowed.md"}, - {"id": "kept2", "source_file": "allowed.md"}, - {"id": "stray", "source_file": "outside.md"}, - ] - hyperedges = [ - {"id": "he_bad", "nodes": ["kept", "stray"], "source_file": "allowed.md"}, - {"id": "he_ok", "nodes": ["kept", "kept2"], "source_file": "allowed.md"}, - ] - with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): - save_semantic_cache( - nodes, [], hyperedges, root=tmp_path, allowed_source_files=["allowed.md"] - ) - - _, _, cached_hyperedges, uncached = check_semantic_cache( - [str(allowed)], root=tmp_path - ) - assert uncached == [] - assert {h["id"] for h in cached_hyperedges} == {"he_ok"} - - -def test_save_semantic_cache_unscoped_preserves_dangling_refs_verbatim(tmp_path): - """#1916 guard-rail: unscoped callers (allowed_source_files=None) must stay - byte-identical — no pruning happens even when an edge or hyperedge - references a node grouped under a ghost file.""" - from graphify.cache import save_semantic_cache - - doc = tmp_path / "doc.md" - doc.write_text("# Doc\n") - - nodes = [ - {"id": "a", "source_file": "doc.md"}, - {"id": "ghost_n", "source_file": "ghost.md"}, # skipped group (no file) - ] - edges = [{"source": "a", "target": "ghost_n", "source_file": "doc.md"}] - hyperedges = [{"id": "he", "nodes": ["a", "ghost_n"], "source_file": "doc.md"}] - - saved = save_semantic_cache(nodes, edges, hyperedges, root=tmp_path) - assert saved == 1 - - import json - raw = json.loads( - (cache_dir(tmp_path, "semantic") / f"{file_hash(doc, tmp_path)}.json").read_text() - ) - assert raw["edges"] == edges - assert raw["hyperedges"] == hyperedges - - -def test_save_semantic_cache_merge_existing_prunes_only_incoming(tmp_path): - """#1916 + #1715: with merge_existing=True (the llm.py checkpoint path), - only the INCOMING slice is pruned before the union — the prior cached - entry's valid edges must survive untouched.""" - from graphify.cache import save_semantic_cache - - big = tmp_path / "big.md" - big.write_text("# Big\n") - other = tmp_path / "other.md" - other.write_text("# Other\n") - - # checkpoint 1: a clean slice + cache=cache, + ) + + entry = next(iter(cache.values())) + assert entry["result"]["nodes"][0]["source_file"] == "src/sample.py" + assert not (tmp_path / "graphify-out" / "cache").exists() + + +def test_semantic_cache_is_scoped_by_mode_and_prompt(tmp_path: Path) -> None: + source = tmp_path / "doc.md" + source.write_text("# Doc\nBody\n", encoding="utf-8") + result = [{"id": "doc", "source_file": "doc.md"}] + cache: dict = {} + + assert save_semantic_cache( + result, [], root=tmp_path, allowed_source_files=[source], + mode="deep", prompt="prompt-a", cache=cache, + ) == 1 + nodes, _, _, misses = check_semantic_cache( + [str(source)], cache, root=tmp_path, mode="deep", prompt="prompt-a" + ) + assert [node["id"] for node in nodes] == ["doc"] + assert nodes[0]["source_file"] == str(source.resolve()) + assert misses == [] + assert check_semantic_cache( + [str(source)], cache, root=tmp_path, mode=None, prompt="prompt-a" + )[3] == [str(source)] + assert check_semantic_cache( + [str(source)], cache, root=tmp_path, mode="deep", prompt="prompt-b" + )[3] == [str(source)] + + +def test_partial_semantic_entry_is_a_miss(tmp_path: Path) -> None: + source = tmp_path / "doc.md" + source.write_text("body\n", encoding="utf-8") + cache: dict = {} save_semantic_cache( - [{"id": "a", "source_file": "big.md"}], - [{"source": "a", "target": "a", "relation": "self", "source_file": "big.md"}], - root=tmp_path, - merge_existing=True, - allowed_source_files=["big.md"], - ) - # checkpoint 2: incoming slice with a dangling edge to an out-of-scope node - nodes2 = [ - {"id": "b", "source_file": "big.md"}, - {"id": "stray", "source_file": "other.md"}, - ] - edges2 = [ - {"source": "b", "target": "stray", "source_file": "big.md"}, - {"source": "a", "target": "b", "source_file": "big.md"}, - ] - with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): - save_semantic_cache( - nodes2, edges2, root=tmp_path, merge_existing=True, - allowed_source_files=["big.md"], - ) - - cached = load_cached(big, root=tmp_path, kind="semantic") - assert {n["id"] for n in cached["nodes"]} == {"a", "b"} - pairs = [(e["source"], e["target"]) for e in cached["edges"]] - assert ("a", "a") in pairs, "prior entry's valid edge must survive the union" - assert ("a", "b") in pairs, "incoming valid edge must be kept" - assert not any("stray" in p for p in pairs) - - -# --- extraction-prompt fingerprinting (#1939) ------------------------------- - - -def test_prompt_fingerprint_stable_and_prompt_sensitive(tmp_path): - """The fingerprint is stable for identical prompts and differs when the - prompt text changes — the whole invalidation signal rests on this.""" - from graphify.cache import prompt_fingerprint - - assert prompt_fingerprint("extract a graph") == prompt_fingerprint("extract a graph") - assert prompt_fingerprint("extract a graph") != prompt_fingerprint("extract a graph v2") - - # A Path is read and hashed as its contents, so the skill path (which loads - # references/extraction-spec.md) and the Python path agree on the same text. - spec = tmp_path / "extraction-spec.md" - spec.write_text("extract a graph", encoding="utf-8") - assert prompt_fingerprint(spec) == prompt_fingerprint("extract a graph") - - -def test_prompt_fingerprint_ignores_line_endings(tmp_path): - """A CRLF checkout of the same spec must not look like a prompt change — - otherwise every Windows run re-bills the whole corpus.""" - from graphify.cache import prompt_fingerprint - - assert prompt_fingerprint("a\r\nb\r\n") == prompt_fingerprint("a\nb\n") - assert prompt_fingerprint("a \nb\n") == prompt_fingerprint("a\nb\n") - - -def test_semantic_cache_prompt_change_invalidates(tmp_path): - """The reported bug (#1939): after the extraction prompt changes, an - unchanged file must MISS instead of replaying the older vintage.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n\nBody.\n") - save_semantic_cache([{"id": "old_vintage", "source_file": "doc.md"}], [], - root=tmp_path, prompt="PROMPT V1") - - # Same prompt: hit. - nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V1") - assert [n["id"] for n in nodes] == ["old_vintage"] - assert uncached == [] - - # Prompt changed (an upgrade shipped a new extraction-spec): must re-extract. - nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V2") - assert nodes == [] - assert uncached == [str(f)], "a new prompt must not replay the old prompt's entry" - - # V2's results land in their own namespace and do not clobber V1's, so - # rolling back to V1 still hits rather than re-billing. - save_semantic_cache([{"id": "new_vintage", "source_file": "doc.md"}], [], - root=tmp_path, prompt="PROMPT V2") - nodes, _, _, _ = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V2") - assert [n["id"] for n in nodes] == ["new_vintage"] - nodes, _, _, _ = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V1") - assert [n["id"] for n in nodes] == ["old_vintage"] - - -def test_semantic_cache_prompt_namespaced_layout(tmp_path): - """Fingerprinted entries live under cache/semantic/p{fp}/, never flat.""" - from graphify.cache import prompt_fingerprint, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], - root=tmp_path, prompt="PROMPT V1") - - sem = tmp_path / "graphify-out" / "cache" / "semantic" - h = file_hash(f, tmp_path) - assert (sem / f"p{prompt_fingerprint('PROMPT V1')}" / f"{h}.json").exists() - assert not (sem / f"{h}.json").exists(), ( - "a known-vintage entry must never be written into the flat unknown-vintage layout" + [{"id": "doc", "source_file": "doc.md"}], [], root=tmp_path, + allowed_source_files=[source], partial_source_files=["doc.md"], cache=cache, ) + assert check_semantic_cache([str(source)], cache, root=tmp_path)[3] == [str(source)] -def test_semantic_cache_prompt_and_mode_compose(tmp_path): - """The prompt fingerprint nests inside the deep namespace (#1894), so the - two dimensions are independent.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], - root=tmp_path, mode="deep", prompt="PROMPT V1") - - deep = tmp_path / "graphify-out" / "cache" / "semantic-deep" - assert list(deep.glob("p*/*.json")), "deep + prompt must nest under semantic-deep/p{fp}/" - - # Right mode, wrong prompt -> miss. Right prompt, wrong mode -> miss. - _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, mode="deep", - prompt="PROMPT V2") - assert uncached == [str(f)] - _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V1") - assert uncached == [str(f)] - # Both right -> hit. - nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, mode="deep", - prompt="PROMPT V1") - assert [n["id"] for n in nodes] == ["d"] and uncached == [] - - -def test_semantic_cache_legacy_entries_served_with_warning(tmp_path): - """Entries written before fingerprinting have unknowable vintage. They are - still served — dropping them would re-bill a whole corpus on upgrade — but - the user is told how many, which is the signal #1939 says is missing today.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - a = tmp_path / "a.md" - a.write_text("# A\n") - b = tmp_path / "b.md" - b.write_text("# B\n") - # Pre-fingerprint writes: the historical flat layout. - save_semantic_cache([{"id": "a_old", "source_file": "a.md"}, - {"id": "b_old", "source_file": "b.md"}], [], root=tmp_path) - - with pytest.warns(RuntimeWarning, match="2 semantic cache entries predate"): - nodes, _, _, uncached = check_semantic_cache( - [str(a), str(b)], root=tmp_path, prompt="PROMPT V1" - ) - assert {n["id"] for n in nodes} == {"a_old", "b_old"} - assert uncached == [] - - -def test_semantic_cache_fingerprinted_entry_beats_legacy(tmp_path): - """Once a file is re-extracted under the current prompt, its fingerprinted - entry wins and the stale flat one is no longer consulted (no warning).""" - import warnings as _warnings - from graphify.cache import check_semantic_cache, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "unknown_vintage", "source_file": "doc.md"}], [], - root=tmp_path) # legacy flat - save_semantic_cache([{"id": "current", "source_file": "doc.md"}], [], - root=tmp_path, prompt="PROMPT V1") - - with _warnings.catch_warnings(): - _warnings.simplefilter("error") # any legacy warning would raise here - nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, - prompt="PROMPT V1") - assert [n["id"] for n in nodes] == ["current"] - assert uncached == [] - - -def test_semantic_cache_merge_existing_never_fuses_legacy_vintage(tmp_path): - """merge_existing must not union a pre-fingerprint entry into a write it is - about to stamp as current-vintage — that would mix two prompts inside one - entry and then attest the result to a prompt that produced half of it.""" - from graphify.cache import load_cached, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "unknown_vintage", "source_file": "doc.md"}], [], - root=tmp_path) # legacy flat - save_semantic_cache([{"id": "current", "source_file": "doc.md"}], [], - root=tmp_path, merge_existing=True, prompt="PROMPT V1") - - entry = load_cached(f, root=tmp_path, kind="semantic", prompt="PROMPT V1") - assert [n["id"] for n in entry["nodes"]] == ["current"] - - # Within one prompt, merge_existing still unions across checkpoints. - save_semantic_cache([{"id": "second_chunk", "source_file": "doc.md"}], [], - root=tmp_path, merge_existing=True, prompt="PROMPT V1") - entry = load_cached(f, root=tmp_path, kind="semantic", prompt="PROMPT V1") - assert {n["id"] for n in entry["nodes"]} == {"current", "second_chunk"} - - -def test_semantic_prune_and_clear_reach_fingerprint_subdirs(tmp_path): - """A glob that stopped at the top level would leave every fingerprinted - entry unprunable, re-growing the unbounded-orphan problem of #1527.""" - from graphify.cache import ( - cached_files, clear_cache, prune_semantic_cache, save_semantic_cache, - ) - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], - root=tmp_path, prompt="PROMPT V1") - h = file_hash(f, tmp_path) - assert h in cached_files(tmp_path), "cached_files must see fingerprinted entries" - - # Live: kept. - assert prune_semantic_cache(tmp_path, {h}) == 0 - # Orphaned (content changed / file deleted): pruned. - assert prune_semantic_cache(tmp_path, set()) == 1 - - save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], - root=tmp_path, prompt="PROMPT V1") - clear_cache(tmp_path) - assert not list((tmp_path / "graphify-out" / "cache" / "semantic").glob("**/*.json")) - - -def test_semantic_cache_unreadable_prompt_file_warns_and_falls_back(tmp_path): - """A skill snippet substitutes SPEC_PATH by hand. If it lands on a path that - isn't there, the fallback to the unattributed layout must be loud: silently - reverting to unversioned keying is exactly the #1939 behavior being fixed.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], root=tmp_path) - - with pytest.warns(RuntimeWarning, match="could not read extraction prompt"): - nodes, _, _, uncached = check_semantic_cache( - [str(f)], root=tmp_path, prompt_file=str(tmp_path / "nope.md") - ) - # Fell back rather than aborting the run. - assert [n["id"] for n in nodes] == ["n"] and uncached == [] - - -def test_prompt_file_reflects_edited_spec(tmp_path): - """The prompt-file fingerprint is memoized per (path, size, mtime); an edited - spec must still register as a new prompt rather than reusing a stale memo.""" - from graphify.cache import check_semantic_cache, save_semantic_cache - - spec = tmp_path / "extraction-spec.md" - spec.write_text("prompt one", encoding="utf-8") - f = tmp_path / "doc.md" - f.write_text("# Doc\n") - - save_semantic_cache([{"id": "v1", "source_file": "doc.md"}], [], - root=tmp_path, prompt_file=str(spec)) - nodes, _, _, _ = check_semantic_cache([str(f)], root=tmp_path, prompt_file=str(spec)) - assert [n["id"] for n in nodes] == ["v1"] - - # An upgrade rewrites the spec: the same file path is now a different prompt. - import os as _os - spec.write_text("prompt two — rewritten by an upgrade", encoding="utf-8") - _os.utime(spec, ns=(0, 0)) # force a distinct stat signature - _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, prompt_file=str(spec)) - assert uncached == [str(f)], "an edited spec must invalidate, not reuse the memo" +def test_cached_files_clear_and_prune(tmp_path: Path) -> None: + ast = tmp_path / "a.py" + semantic = tmp_path / "b.md" + ast.write_text("pass\n", encoding="utf-8") + semantic.write_text("body\n", encoding="utf-8") + cache: dict = {} + save_cached(ast, {"nodes": [], "edges": []}, root=tmp_path, cache=cache) + save_semantic_cache( + [{"id": "b", "source_file": "b.md"}], [], root=tmp_path, + allowed_source_files=[semantic], cache=cache, + ) + hashes = cached_files(cache) + assert file_hash(ast) in hashes + assert file_hash(semantic) in hashes + assert prune_semantic_cache(cache, {"not-live"}) == 1 + assert len(cache) == 1 + clear_cache(cache) + assert cache == {} + + +def test_markdown_frontmatter_only_change_keeps_hash(tmp_path: Path) -> None: + source = tmp_path / "doc.md" + source.write_text("---\nreviewed: one\n---\n\nBody", encoding="utf-8") + first = file_hash(source) + source.write_text("---\nreviewed: two\n---\n\nBody", encoding="utf-8") + assert file_hash(source) == first + source.write_text("---\nreviewed: two\n---\n\nChanged", encoding="utf-8") + assert file_hash(source) != first + + +def test_frontmatter_delimiters_must_be_whole_lines() -> None: + content = b"----\nIntro\n---\nbody" + assert _body_content(content) == content + assert _body_content(b"---\ntitle: Test\n---\nbody") == b"\nbody" + assert _body_content(b"---\ntitle: Test\n--- not close\nbody") == b"---\ntitle: Test\n--- not close\nbody" diff --git a/tests/test_callflow_html.py b/tests/test_callflow_html.py index 9605c9ba1..1b5a070f7 100644 --- a/tests/test_callflow_html.py +++ b/tests/test_callflow_html.py @@ -1,187 +1,83 @@ -import json import subprocess import sys from pathlib import Path -from graphify.callflow_html import derive_sections_from_communities, write_callflow_html +import pytest +from graphify.callflow_html import derive_sections_from_communities, load_graph, write_callflow_html +from graphify.helix.state import new_state +from tests.native_helpers import make_loaded -def _make_graphify_out(tmp_path: Path) -> Path: - out = tmp_path / "graphify-out" - out.mkdir() - graph = { - "directed": False, - "multigraph": False, - "graph": {}, - "nodes": [ - {"id": "api", "label": "ApiClient", "source_file": "src/api.py", "file_type": "code", "community": 0}, - {"id": "run", "label": "run()", "source_file": "src/main.py", "file_type": "code", "community": 0}, - {"id": "export", "label": "write_html()", "source_file": "src/export.py", "file_type": "code", "community": 1}, - {"id": "evil", "label": "", "source_file": "src/evil.py", "file_type": "code", "community": 1}, - ], - "links": [ - {"source": "run", "target": "api", "relation": "calls", "confidence": "EXTRACTED", "confidence_score": 1.0}, - {"source": "api", "target": "export", "relation": "uses", "confidence": "EXTRACTED", "confidence_score": 1.0}, - {"source": "export", "target": "evil", "relation": "calls", "confidence": "EXTRACTED", "confidence_score": 1.0}, + +def _make_graphify_out(root: Path, *, external: bool = False) -> Path: + out = root / "graphify-out" + out.mkdir(parents=True) + prefix = "External" if external else "" + nodes = [ + {"id": "api", "label": prefix + "ApiClient", "source_file": "src/api.py", "file_type": "code"}, + {"id": "run", "label": prefix + "run()", "source_file": "src/main.py", "file_type": "code"}, + {"id": "export", "label": prefix + "write_html()", "source_file": "src/export.py", "file_type": "code"}, + {"id": "evil", "label": "", "source_file": "src/evil.py", "file_type": "code"}, + ] + state = new_state(communities=[ + {"id": 0, "members": ["api", "run"], "name": prefix + "Runtime", "cohesion": 0.8}, + {"id": 1, "members": ["export", "evil"], "name": prefix + "Export", "cohesion": 0.7}, + ]) + make_loaded( + out, + nodes=nodes, + edges=[ + {"source": "run", "target": "api", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "api", "target": "export", "relation": "uses", "confidence": "EXTRACTED"}, + {"source": "export", "target": "evil", "relation": "calls", "confidence": "EXTRACTED"}, ], - "hyperedges": [], - "built_at_commit": "abcdef123456", - } - (out / "graph.json").write_text(json.dumps(graph), encoding="utf-8") - (out / ".graphify_labels.json").write_text( - json.dumps({"0": "Runtime", "1": "Export"}), - encoding="utf-8", + state=state, ) (out / "GRAPH_REPORT.md").write_text( - "\n".join( - [ - "# Graph Report - sample", - "", - "## Summary", - "- 3 nodes · 2 edges · 1 communities detected", - "", - "## God Nodes (most connected - your core abstractions)", - "1. `Transformer` - 2 edges", - ] - ), + "# Graph Report\n\n## God Nodes\n1. `Transformer` - 2 edges\n", encoding="utf-8", ) return out -def test_write_callflow_html_creates_file_and_uses_report(tmp_path): +def test_write_callflow_html_uses_native_store_and_report(tmp_path): out = _make_graphify_out(tmp_path) - - html_path = write_callflow_html( - tmp_path, - output="graphify-out/callflow.html", - max_sections=4, - ) - - assert html_path == out / "callflow.html" - content = html_path.read_text(encoding="utf-8") - assert "mermaid" in content - assert "Graph Report Highlights" in content - assert "Transformer" in content - assert "ApiClient" in content + path = write_callflow_html(tmp_path, output=out / "callflow.html", max_sections=4) + content = path.read_text(encoding="utf-8") + assert "mermaid" in content and "Transformer" in content and "ApiClient" in content assert "<script>alert(1)</script>" in content assert "" not in content -def test_export_callflow_html_cli_creates_file(tmp_path): - _make_graphify_out(tmp_path) - +def test_export_callflow_cli_accepts_default_and_positional_native_store(tmp_path): + out = _make_graphify_out(tmp_path) result = subprocess.run( - [ - sys.executable, - "-m", - "graphify", - "export", - "callflow-html", - "--output", - "graphify-out/from-cli.html", - "--max-sections", - "4", - ], - cwd=tmp_path, - capture_output=True, - text=True, + [sys.executable, "-m", "graphify", "export", "callflow-html", "--output", str(out / "default.html")], + cwd=tmp_path, capture_output=True, text=True, ) - assert result.returncode == 0, result.stderr - html_path = tmp_path / "graphify-out" / "from-cli.html" - assert html_path.exists() - assert "callflow HTML written" in result.stdout - - -def test_export_callflow_html_cli_accepts_positional_graph_path(tmp_path): - _make_graphify_out(tmp_path) - external_out = tmp_path / "GitNexus" / "graphify-out" - external_out.mkdir(parents=True) - graph = { - "directed": False, - "multigraph": False, - "graph": {}, - "nodes": [ - {"id": "external", "label": "ExternalOnly", "source_file": "src/external.py", "file_type": "code", "community": 0}, - {"id": "writer", "label": "write_external()", "source_file": "src/writer.py", "file_type": "code", "community": 1}, - ], - "links": [ - {"source": "external", "target": "writer", "relation": "calls", "confidence": "EXTRACTED", "confidence_score": 1.0}, - ], - "hyperedges": [], - } - (external_out / "graph.json").write_text(json.dumps(graph), encoding="utf-8") - (external_out / ".graphify_labels.json").write_text(json.dumps({"0": "External Runtime", "1": "External Export"}), encoding="utf-8") - (external_out / "GRAPH_REPORT.md").write_text( - "\n".join( - [ - "# Graph Report - external", - "", - "## Summary", - "- 2 nodes · 1 edges · 2 communities detected", - "", - "## God Nodes (most connected - your core abstractions)", - "1. `ExternalGod` - 1 edges", - ] - ), - encoding="utf-8", - ) - - result = subprocess.run( - [ - sys.executable, - "-m", - "graphify", - "export", - "callflow-html", - str(external_out / "graph.json"), - "--output", - "positional.html", - "--max-sections", - "4", - ], - cwd=tmp_path, - capture_output=True, - text=True, + external = _make_graphify_out(tmp_path / "external", external=True) + positional = subprocess.run( + [sys.executable, "-m", "graphify", "export", "callflow-html", str(external / "graph.helix"), "--output", str(tmp_path / "positional.html")], + cwd=tmp_path, capture_output=True, text=True, ) - - assert result.returncode == 0, result.stderr - html = (tmp_path / "positional.html").read_text(encoding="utf-8") - assert "ExternalOnly" in html - assert "ExternalGod" in html - assert "ApiClient" not in html - assert "Transformer" not in html + assert positional.returncode == 0, positional.stderr + html = (tmp_path / "positional.html").read_text() + assert "ExternalApiClient" in html and "ApiClient" in html def test_derive_sections_groups_by_architecture_keywords(): nodes = [ - {"id": "extract_py", "label": "extract_python", "source_file": "graphify/extract.py", "community": 0}, - {"id": "extract_js", "label": "extract_js", "source_file": "graphify/extract.py", "community": 0}, - {"id": "to_html", "label": "to_html", "source_file": "graphify/export.py", "community": 1}, - {"id": "test_html", "label": "test_export_html", "source_file": "tests/test_export.py", "community": 2}, + {"id": "extract", "label": "extract_python", "source_file": "graphify/extract.py", "community": 0}, + {"id": "html", "label": "to_html", "source_file": "graphify/export.py", "community": 1}, + {"id": "test", "label": "test_export_html", "source_file": "tests/test_export.py", "community": 2}, ] - - sections = derive_sections_from_communities(nodes, {}, "en", 6) - ids = {section["id"] for section in sections} - - assert "extract-pipeline" in ids - assert "outputs-docs" in ids - assert "tests-fixtures" in ids + ids = {section["id"] for section in derive_sections_from_communities(nodes, {}, "en", 6)} + assert {"extract-pipeline", "outputs-docs", "tests-fixtures"} <= ids -def test_load_graph_rejects_oversized_file(monkeypatch, tmp_path): - """#F4: callflow_html.load_graph must refuse to read a graph.json that - exceeds the size cap (SystemExit via translated ValueError).""" - import pytest - from graphify.callflow_html import load_graph - - graph_path = tmp_path / "graph.json" - graph_path.write_text( - json.dumps({"nodes": [], "links": []}), - encoding="utf-8", - ) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 8) - with pytest.raises(SystemExit) as excinfo: - load_graph(graph_path) - assert "exceeds" in str(excinfo.value) +def test_load_graph_rejects_legacy_json(tmp_path): + legacy = tmp_path / "graph.json" + legacy.write_text("{}") + with pytest.raises((ValueError, FileNotFoundError)): + load_graph(legacy) diff --git a/tests/test_chunking.py b/tests/test_chunking.py index dea9f68f2..eff14f3eb 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -207,7 +207,7 @@ def test_corpus_parallel_merge_order_is_submission_order_not_completion(tmp_path """#1632: merged node/edge order must be deterministic (submission order), not the order chunks' network calls happen to finish. We skew latencies so the first-submitted chunk finishes LAST; the merged result must still be in - file/submission order so graph.json is stable run-to-run.""" + file/submission order so the graph is stable run-to-run.""" from graphify.llm import extract_corpus_parallel files = [] @@ -289,13 +289,14 @@ def test_checkpoint_scopes_cache_writes_to_chunk_files(tmp_path): a = tmp_path / "A.py"; a.write_text("def a(): pass") b = tmp_path / "B.py"; b.write_text("def b(): pass") + cache = {} # Seed B.py's legitimate semantic cache (a full, correct entry). save_semantic_cache( [{"id": "b_real", "source_file": "B.py", "file_type": "code"}], - [], [], root=tmp_path, + [], [], root=tmp_path, cache=cache, ) - before = load_cached(b, tmp_path, kind="semantic") + before = load_cached(b, tmp_path, kind="semantic", cache=cache) assert before and [n["id"] for n in before["nodes"]] == ["b_real"] # The chunk dispatches only A.py, but the (untrusted) model result attributes @@ -314,10 +315,11 @@ def stray(chunk, **kwargs): extract_corpus_parallel( [a], backend="kimi", root=tmp_path, token_budget=None, chunk_size=1, max_concurrency=1, + cache=cache, ) # B.py's cache is unchanged: the stray node was rejected, not merged in. - after = load_cached(b, tmp_path, kind="semantic") + after = load_cached(b, tmp_path, kind="semantic", cache=cache) assert [n["id"] for n in after["nodes"]] == ["b_real"], ( f"B.py cache was clobbered by an out-of-chunk node: {after}" ) @@ -325,7 +327,7 @@ def stray(chunk, **kwargs): # entries with the prompt that produced them (#1939), so read that namespace. from graphify.llm import _extraction_system - a_cache = load_cached(a, tmp_path, kind="semantic", prompt=_extraction_system()) + a_cache = load_cached(a, tmp_path, kind="semantic", prompt=_extraction_system(), cache=cache) assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"]) @@ -337,6 +339,7 @@ def test_truncated_chunk_is_cached_partial_and_missed_on_reload(tmp_path): from graphify.cache import load_cached doc = tmp_path / "doc.md"; doc.write_text("# Heading\nlots of prose\n") + cache = {} def truncated(chunk, **kwargs): return { @@ -350,10 +353,12 @@ def truncated(chunk, **kwargs): extract_corpus_parallel( [doc], backend="kimi", root=tmp_path, token_budget=None, chunk_size=1, max_concurrency=1, + cache=cache, ) # The entry was written but stamped partial, so load_cached rejects it. - assert load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system()) is None + assert load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system(), cache=cache) is None + assert any(entry.get("partial") for entry in cache.values()) def test_checkpoint_writes_deep_namespace_in_deep_mode(tmp_path): @@ -365,6 +370,7 @@ def test_checkpoint_writes_deep_namespace_in_deep_mode(tmp_path): doc = tmp_path / "doc.md" doc.write_text("# Doc\n\nsome content\n") + cache = {} def ok(chunk, **kwargs): return { @@ -377,17 +383,18 @@ def ok(chunk, **kwargs): [doc], backend="kimi", root=tmp_path, token_budget=None, chunk_size=1, max_concurrency=1, deep_mode=True, + cache=cache, ) # The checkpoint also stamps entries with the prompt that produced them # (#1939) — a deep run's prompt carries the deep suffix. deep = load_cached(doc, tmp_path, kind="semantic-deep", - prompt=_extraction_system(deep=True)) + prompt=_extraction_system(deep=True), cache=cache) assert deep and [n["id"] for n in deep["nodes"]] == ["d1"], ( "deep-mode checkpoint must land in cache/semantic-deep/" ) assert load_cached(doc, tmp_path, kind="semantic", - prompt=_extraction_system(deep=False)) is None, ( + prompt=_extraction_system(deep=False), cache=cache) is None, ( "deep-mode checkpoint must not write the standard semantic namespace" ) @@ -429,7 +436,7 @@ def omit_odd(chunk, **kwargs): def test_out_of_scope_nodes_are_dropped_from_merged_result(tmp_path, capsys): """#1895: the #1757 cache guard skips the CACHE write for a node attributed to a real corpus file that was not dispatched, but the node itself still - flowed into merged["nodes"] and landed in graph.json. The merged result must + flowed into merged["nodes"] and landed in the durable graph. The merged result must drop such nodes (and edges/hyperedges touching them), warn once, and record the count — while keeping in-scope sibling attributions (a node attributed to a different dispatched file in the same chunk) and non-file concept @@ -528,6 +535,7 @@ def test_checkpoint_caches_sliced_document_chunks(tmp_path, capsys): doc.write_text("# Title\n" + ("word " * 12000) + "\n## Section\n" + ("more " * 12000)) # sanity: the doc really does slice into FileSlice units units = expand_oversized_files([doc], _FILE_CHAR_CAP) + cache = {} assert len(units) > 1 and all(isinstance(u, FileSlice) for u in units) def sliced(chunk, **kwargs): @@ -541,13 +549,14 @@ def sliced(chunk, **kwargs): extract_corpus_parallel( [doc], backend="kimi", root=tmp_path, token_budget=None, chunk_size=1, max_concurrency=1, + cache=cache, ) assert "incremental cache checkpoint failed" not in capsys.readouterr().err, ( "checkpoint raised on a FileSlice chunk (#1870)" ) # The checkpoint stamps entries with the prompt that produced them (#1939). - cached = load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system()) + cached = load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system(), cache=cache) assert cached and any(n["id"] == "big_title" for n in cached["nodes"]), ( "sliced document was never checkpointed (#1870)" ) diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 6173a61aa..9d8ad646f 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -1,543 +1,102 @@ -"""Integration tests for graphify export subcommands and CLI commands. +"""End-to-end native CLI query and presentation export tests.""" -Each test builds a minimal graph in a temp dir, runs the CLI command as a subprocess, -and asserts the expected output file exists and is non-empty / valid. -""" -from __future__ import annotations -import json import os import subprocess import sys from pathlib import Path -import pytest +from graphify.helix.state import new_state +from tests.native_helpers import make_loaded -PYTHON = sys.executable -FIXTURES = Path(__file__).parent / "fixtures" - -def _run(args: list[str], cwd: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess: - return subprocess.run( - [PYTHON, "-m", "graphify"] + args, - cwd=cwd, - capture_output=True, - text=True, - env=env, - ) - - -def _make_graph(tmp_path: Path) -> Path: - """Build a minimal graph.json + analysis/labels files in tmp_path/graphify-out/.""" +def _project(tmp_path: Path) -> Path: out = tmp_path / "graphify-out" - out.mkdir() - - extraction = json.loads((FIXTURES / "extraction.json").read_text()) - from graphify.build import build_from_json - from graphify.cluster import cluster, score_all - from graphify.analyze import god_nodes, surprising_connections - from graphify.export import to_json - - G = build_from_json(extraction) - communities = cluster(G) - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - labels = {cid: f"Community {cid}" for cid in communities} - - to_json(G, communities, str(out / "graph.json")) - - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - } - (out / ".graphify_analysis.json").write_text(json.dumps(analysis)) - (out / ".graphify_labels.json").write_text( - json.dumps({str(k): v for k, v in labels.items()}) - ) - return out - - -# ── graphify export html ───────────────────────────────────────────────────── - -def test_export_html_creates_file(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "html"], tmp_path) - assert r.returncode == 0, r.stderr - html = tmp_path / "graphify-out" / "graph.html" - assert html.exists() - assert html.stat().st_size > 0 - - -def test_export_html_no_viz_removes_file(tmp_path): - out = _make_graph(tmp_path) - (out / "graph.html").write_text("") - r = _run(["export", "html", "--no-viz"], tmp_path) - assert r.returncode == 0, r.stderr - assert not (out / "graph.html").exists() - - -def test_export_html_error_without_graph(tmp_path): - r = _run(["export", "html"], tmp_path) - assert r.returncode != 0 - - -# ── graphify export obsidian ───────────────────────────────────────────────── - -def test_export_obsidian_creates_vault(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "obsidian"], tmp_path) - assert r.returncode == 0, r.stderr - vault = tmp_path / "graphify-out" / "obsidian" - assert vault.exists() - md_files = list(vault.glob("*.md")) - assert len(md_files) > 0 - - -def test_export_obsidian_custom_dir(tmp_path): - _make_graph(tmp_path) - custom = tmp_path / "my-vault" - r = _run(["export", "obsidian", "--dir", str(custom)], tmp_path) - assert r.returncode == 0, r.stderr - assert custom.exists() - assert len(list(custom.glob("*.md"))) > 0 - - -# ── graphify export wiki ───────────────────────────────────────────────────── - -def test_export_wiki_creates_articles(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "wiki"], tmp_path) - assert r.returncode == 0, r.stderr - wiki = tmp_path / "graphify-out" / "wiki" - assert wiki.exists() - assert (wiki / "index.md").exists() - - -def test_export_wiki_accepts_edges_only_graph_json(tmp_path): - out = _make_graph(tmp_path) - graph_path = out / "graph.json" - data = json.loads(graph_path.read_text()) - data["edges"] = data.pop("links") - graph_path.write_text(json.dumps(data)) - - r = _run(["export", "wiki"], tmp_path) - - assert r.returncode == 0, r.stderr - assert (out / "wiki" / "index.md").exists() - - -# ── graphify export graphml ────────────────────────────────────────────────── - -def test_export_graphml_creates_file(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "graphml"], tmp_path) - assert r.returncode == 0, r.stderr - gml = tmp_path / "graphify-out" / "graph.graphml" - assert gml.exists() - assert gml.stat().st_size > 0 - content = gml.read_text() - assert " 0 - content = cypher.read_text() - assert "MERGE" in content or "CREATE" in content - - -# ── graphify export falkordb (cypher) ──────────────────────────────────────── - -def test_export_falkordb_creates_cypher(tmp_path): - _make_graph(tmp_path) - r = _run(["export", "falkordb"], tmp_path) - assert r.returncode == 0, r.stderr - cypher = tmp_path / "graphify-out" / "cypher.txt" - assert cypher.exists() - assert cypher.stat().st_size > 0 - content = cypher.read_text() - assert "MERGE" in content or "CREATE" in content - - -# ── graphify query ─────────────────────────────────────────────────────────── - -def test_query_returns_output(tmp_path): - _make_graph(tmp_path) - r = _run(["query", "test"], tmp_path) - assert r.returncode == 0, r.stderr - assert len(r.stdout) > 0 - - -def test_query_dfs_flag(tmp_path): - _make_graph(tmp_path) - r = _run(["query", "test", "--dfs"], tmp_path) - assert r.returncode == 0, r.stderr - - -def test_query_budget_flag(tmp_path): - _make_graph(tmp_path) - r = _run(["query", "test", "--budget", "500"], tmp_path) - assert r.returncode == 0, r.stderr - - -def test_query_missing_graph_fails(tmp_path): - r = _run(["query", "anything"], tmp_path) - assert r.returncode != 0 - - -def test_query_uses_graphify_out_env(tmp_path): - out = _make_graph(tmp_path) - custom_out = tmp_path / "custom-graph" - out.rename(custom_out) - env = os.environ.copy() - env["GRAPHIFY_OUT"] = custom_out.name - - r = _run(["query", "test"], tmp_path, env=env) - - assert r.returncode == 0, r.stderr - assert len(r.stdout) > 0 - - -def test_extract_writes_to_graphify_out_env(tmp_path): - """#1423: `graphify extract` honours GRAPHIFY_OUT for where it WRITES, not only - where readers look — previously it hardcoded graphify-out/ and ignored the - override. Code-only corpus, so no LLM backend is needed.""" - (tmp_path / "m.py").write_text("def a():\n return b()\n\n\ndef b():\n return 1\n") - env = os.environ.copy() - env["GRAPHIFY_OUT"] = "custom-out" - - r = _run(["extract", "."], tmp_path, env=env) - - assert r.returncode == 0, r.stderr - assert (tmp_path / "custom-out" / "graph.json").exists(), r.stdout - assert (tmp_path / "custom-out" / "manifest.json").exists() - # The default dir must NOT be created when the override is set. - assert not (tmp_path / "graphify-out").exists(), "extract ignored GRAPHIFY_OUT and wrote graphify-out/" - # Manifest keys are relative to the scan root (portable) — #1417. - keys = list(json.loads((tmp_path / "custom-out" / "manifest.json").read_text()).keys()) - assert keys == ["m.py"], keys - - -# ── graphify path ──────────────────────────────────────────────────────────── - -def test_path_runs_without_error(tmp_path): - _make_graph(tmp_path) - r = _run(["path", "Transformer", "LayerNorm"], tmp_path) - # May find or not find a path — either is valid, should not crash - assert r.returncode == 0, r.stderr - - -def test_path_missing_graph_fails(tmp_path): - r = _run(["path", "a", "b"], tmp_path) - assert r.returncode != 0 - - -def test_path_uses_graphify_out_env(tmp_path): - out = _make_graph(tmp_path) - custom_out = tmp_path / "custom-graph" - out.rename(custom_out) - env = os.environ.copy() - env["GRAPHIFY_OUT"] = custom_out.name - - r = _run(["path", "Transformer", "LayerNorm"], tmp_path, env=env) - - assert r.returncode == 0, r.stderr - - -# ── graphify explain ───────────────────────────────────────────────────────── - -def test_explain_runs_without_error(tmp_path): - _make_graph(tmp_path) - r = _run(["explain", "test"], tmp_path) - assert r.returncode == 0, r.stderr - - -def test_explain_missing_graph_fails(tmp_path): - r = _run(["explain", "anything"], tmp_path) - assert r.returncode != 0 - - -def test_explain_uses_graphify_out_env(tmp_path): - out = _make_graph(tmp_path) - custom_out = tmp_path / "custom-graph" - out.rename(custom_out) - env = os.environ.copy() - env["GRAPHIFY_OUT"] = custom_out.name - - r = _run(["explain", "test"], tmp_path, env=env) - - assert r.returncode == 0, r.stderr - - -# ── graphify export unknown format ─────────────────────────────────────────── - -def test_export_unknown_format_fails(tmp_path): - r = _run(["export", "pdf"], tmp_path) - assert r.returncode != 0 - - -def test_update_no_cluster_writes_raw_graph(tmp_path): - src = tmp_path / "sample.py" - src.write_text("def f():\n return 1\n", encoding="utf-8") - - r = _run(["update", ".", "--no-cluster"], tmp_path) - assert r.returncode == 0, r.stderr - - graph_path = tmp_path / "graphify-out" / "graph.json" - assert graph_path.exists() - data = json.loads(graph_path.read_text(encoding="utf-8")) - assert "nodes" in data and "links" in data - assert all("community" not in node for node in data["nodes"]) - - -# Regression test for #934 - cluster-only crashes when graphify-out/ doesn't exist - -def test_cluster_only_creates_output_dir_when_missing(tmp_path): - """cluster-only must not crash with FileNotFoundError when graphify-out/ is absent (#934).""" - # Build graph.json somewhere other than the default graphify-out/ location - # so we can point --graph at it while graphify-out/ doesn't exist yet. - graph_src = tmp_path / "backup" / "graph.json" - graph_src.parent.mkdir() - - out_dir = _make_graph(tmp_path) - graph_json = out_dir / "graph.json" - # Simulate user archiving the output dir before re-clustering - import shutil - shutil.copy(graph_json, graph_src) - shutil.rmtree(out_dir) - - assert not (tmp_path / "graphify-out").exists() - - r = _run(["cluster-only", ".", "--graph", str(graph_src), "--no-viz"], tmp_path) - assert r.returncode == 0, r.stderr - assert (tmp_path / "graphify-out" / "GRAPH_REPORT.md").exists() - - -def test_cluster_only_graph_in_graphify_out_writes_beside_it(tmp_path): - """#1747 Case 2: `cluster-only --graph /graphify-out/graph.json` - must write GRAPH_REPORT.md and the re-clustered graph beside that graph, not - into a stray graphify-out/ in the CWD.""" - project = tmp_path / "project" - project.mkdir() - out_dir = _make_graph(project) # project/graphify-out/graph.json - - cwd = tmp_path / "elsewhere" - cwd.mkdir() - r = _run( - ["cluster-only", ".", "--graph", str(out_dir / "graph.json"), "--no-viz", "--no-label"], - cwd, + out.mkdir(parents=True) + state = new_state(communities=[ + {"id": 0, "members": ["a", "b"], "name": "Runtime", "cohesion": 0.9}, + {"id": 1, "members": ["c"], "name": "Storage", "cohesion": 1.0}, + ]) + make_loaded( + out, + nodes=[ + {"id": "a", "label": "App", "source_file": "app.py", "file_type": "code"}, + {"id": "b", "label": "Service", "source_file": "service.py", "file_type": "code"}, + {"id": "c", "label": "Database", "source_file": "db.py", "file_type": "code"}, + ], + edges=[ + {"source": "a", "target": "b", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "b", "target": "c", "relation": "uses", "confidence": "INFERRED"}, + ], + state=state, ) - assert r.returncode == 0, r.stderr - assert (out_dir / "GRAPH_REPORT.md").exists() # beside --graph - assert not (cwd / "graphify-out").exists() # no CWD pollution + return tmp_path -def test_extract_out_does_not_pollute_corpus(tmp_path): - """#1747 Case 1: `extract --out ` must not leave a stray - graphify-out/ (cache, stat-index) inside the scanned corpus.""" - corpus = tmp_path / "corpus" - corpus.mkdir() - (corpus / "a.py").write_text("def main():\n return 1\n") - out = tmp_path / "scratch" - - r = _run( - ["extract", str(corpus), "--out", str(out), "--no-cluster", "--code-only"], - tmp_path, +def _run(project: Path, *args: str, env=None): + return subprocess.run( + [sys.executable, "-m", "graphify", *args], cwd=project, + capture_output=True, text=True, env=env, ) - assert r.returncode == 0, r.stderr - assert (out / "graphify-out" / "graph.json").exists() # graph in --out - assert not (corpus / "graphify-out").exists() # corpus untouched -# Regression test for #1027 - cluster-only must remap labels via node overlap +def test_html_and_no_viz(tmp_path): + project = _project(tmp_path) + html = tmp_path / "custom.html" + result = _run(project, "export", "html", "--output", str(html)) + assert result.returncode == 0, result.stderr + assert "App" in html.read_text() + result = _run(project, "export", "html", "--output", str(html), "--no-viz") + assert result.returncode == 0 and not html.exists() -def test_cluster_only_persists_analysis_sidecar(tmp_path): - """cluster-only must refresh .graphify_analysis.json alongside graph.json. - Downstream export commands use the sidecar for community membership and - should not see stale or missing community analysis after a recluster. - """ - out = _make_graph(tmp_path) - analysis_path = out / ".graphify_analysis.json" - analysis_path.unlink() - - r = _run(["cluster-only", ".", "--no-viz"], tmp_path) - assert r.returncode == 0, r.stderr - assert analysis_path.exists() - - analysis = json.loads(analysis_path.read_text(encoding="utf-8")) - assert analysis["communities"] - assert analysis["cohesion"] - assert "gods" in analysis - assert "surprises" in analysis - assert "questions" in analysis - - graph = json.loads((out / "graph.json").read_text(encoding="utf-8")) - graph_cids = { - str(node["community"]) - for node in graph.get("nodes", []) - if node.get("community") is not None +def test_obsidian_wiki_graphml_and_cypher(tmp_path): + project = _project(tmp_path) + targets = { + "obsidian": tmp_path / "vault", + "wiki": tmp_path / "wiki", + "graphml": tmp_path / "graph.graphml", + "neo4j": tmp_path / "cypher.txt", } - assert graph_cids == set(analysis["communities"]) - - -def test_cluster_only_remaps_labels_to_previous_cids(tmp_path): - """cluster-only must invoke remap_communities_to_previous so the existing - .graphify_labels.json keeps tracking the same conceptual communities after - re-clustering. Without the remap call, Leiden's size-descending cid order - re-applies labels by raw index and they silently misalign with cluster - contents (#1027). Mirror of the watch/update fix from #822. - """ - out = _make_graph(tmp_path) - graph_json = out / "graph.json" - labels_json = out / ".graphify_labels.json" - - # Tag every node with an out-of-band community id and write a labels file - # keyed on those ids. After cluster-only, at least one of those sentinel - # ids must survive in the labels file (= remap succeeded by node overlap). - # If the cluster-only branch skips remap, Leiden returns small ints - # (0, 1, ...) and the sentinel keys disappear entirely. - g = json.loads(graph_json.read_text(encoding="utf-8")) - nodes = g.get("nodes", []) - assert len(nodes) >= 4, "fixture must have enough nodes to form 2+ communities" - sentinel_a, sentinel_b = 4242, 9999 - half = len(nodes) // 2 - for i, n in enumerate(nodes): - n["community"] = sentinel_a if i < half else sentinel_b - graph_json.write_text(json.dumps(g), encoding="utf-8") - labels_json.write_text( - json.dumps({str(sentinel_a): "First Group", str(sentinel_b): "Second Group"}), - encoding="utf-8", + for kind, target in targets.items(): + result = _run(project, "export", kind, "--output", str(target)) + assert result.returncode == 0, f"{kind}: {result.stderr}" + assert (targets["vault"] if "vault" in targets else targets["obsidian"]).exists() + assert (targets["wiki"] / "index.md").exists() + assert "graphml" in targets["graphml"].read_text() + assert "MERGE" in targets["neo4j"].read_text() + + +def test_query_path_and_explain_read_native_store(tmp_path): + project = _project(tmp_path) + query = _run(project, "query", "Service") + path = _run(project, "path", "App", "Database") + explain = _run(project, "explain", "Service") + assert query.returncode == 0 and "Service" in query.stdout + assert path.returncode == 0 and "App --calls [EXTRACTED]--> Service --uses [INFERRED]--> Database" in path.stdout + assert explain.returncode == 0 and "Node: Service" in explain.stdout + + +def test_positional_native_store_for_export(tmp_path): + project = _project(tmp_path) + output = tmp_path / "positional.graphml" + result = _run( + project, "export", "graphml", str(project / "graphify-out" / "graph.helix"), + "--output", str(output), ) + assert result.returncode == 0, result.stderr + assert output.exists() - r = _run(["cluster-only", ".", "--no-viz"], tmp_path) - assert r.returncode == 0, r.stderr - - # Real signal: labels.json keys must align with the community ids actually - # written to graph.json's per-node community attribute. Without remap, - # Leiden returns small cids (0, 1, ...) but labels.json still carries the - # old sentinel keys, so the intersection is empty and labels are orphaned. - final_graph = json.loads(graph_json.read_text(encoding="utf-8")) - final_labels = json.loads(labels_json.read_text(encoding="utf-8")) - actual_cids = {n.get("community") for n in final_graph.get("nodes", [])} - label_cids = {int(k) for k in final_labels.keys()} - overlap = actual_cids & label_cids - assert overlap, ( - f"After cluster-only with prior labels keyed on cids {label_cids}, at " - f"least one of those cids must still appear in graph.json's community " - f"attribute ({actual_cids}). Without remap_communities_to_previous " - f"(#1027) Leiden renumbers communities to 0,1,... and the prior labels " - f"become orphaned. Final labels: {final_labels}" - ) - - -# ── communities-fallback when .graphify_analysis.json is absent ────────────── -# The watch / post-commit rebuild path only writes graph.json + GRAPH_REPORT.md; -# it does NOT regenerate .graphify_analysis.json. The full `graphify extract` -# pipeline also removes its temp files at the end of the run on some skill -# workflows. In both cases the per-node `community` attribute is intact on -# every node in graph.json — that's the source of truth `to_json` writes. -# Without these tests, `graphify export html|obsidian|wiki|svg|graphml|neo4j` -# silently bails or generates a degraded artifact whenever the sidecar is -# missing, even though the data is right there. - -def test_export_html_falls_back_to_node_community_attribute(tmp_path): - """When .graphify_analysis.json is absent, export html should reconstruct - communities from the per-node attribute in graph.json rather than bailing - out with 'Single community - aggregated view not useful.'. - """ - out = _make_graph(tmp_path) - # Simulate the watch-rebuild / cleanup case: graph.json + labels survive, - # analysis sidecar is gone. - (out / ".graphify_analysis.json").unlink() - r = _run(["export", "html"], tmp_path) - assert r.returncode == 0, r.stderr - html = out / "graph.html" - assert html.exists(), "graph.html should be generated from the fallback" - assert html.stat().st_size > 0 - # The success message comes from to_html — confirm we're not hitting the - # "Single community" bail-out path. - assert "Single community" not in r.stdout - assert "Single community" not in r.stderr - - -def test_export_html_fallback_recovers_multiple_communities(tmp_path): - """Stronger assertion: the reconstructed `communities` dict should have the - SAME community count as the analysis sidecar would, so downstream code - (aggregation thresholds, member counts) sees identical input. - """ - out = _make_graph(tmp_path) - - # Read the canonical community count from the analysis sidecar - analysis = json.loads((out / ".graphify_analysis.json").read_text(encoding="utf-8")) - expected_count = len(analysis["communities"]) - - # And the count we'd reconstruct from graph.json's node attributes - graph = json.loads((out / "graph.json").read_text(encoding="utf-8")) - reconstructed_cids = { - n["community"] for n in graph.get("nodes", []) - if n.get("community") is not None - } - assert len(reconstructed_cids) == expected_count, ( - f"reconstruction would lose communities: sidecar={expected_count} vs " - f"graph.json={len(reconstructed_cids)}" +def test_graphify_out_absolute_override(tmp_path): + project = tmp_path / "project" + project.mkdir() + configured = tmp_path / "shared-output" + configured.mkdir() + make_loaded( + configured, + nodes=[{"id": "x", "label": "OverrideNode", "source_file": "x.py"}], ) - - # Now remove the sidecar and confirm the CLI still succeeds end-to-end. - (out / ".graphify_analysis.json").unlink() - r = _run(["export", "html"], tmp_path) - assert r.returncode == 0, r.stderr - assert (out / "graph.html").exists() - - -def test_export_html_no_community_data_at_all_still_succeeds(tmp_path): - """If a graph.json was somehow written without any per-node `community` - attribute (older versions of to_json, hand-built graphs), the fallback - should produce an empty communities dict and the renderer should still - not crash. Whether the aggregated view is useful is a separate question. - """ - out = _make_graph(tmp_path) - (out / ".graphify_analysis.json").unlink() - - # Strip the community attribute from every node - graph_path = out / "graph.json" - graph = json.loads(graph_path.read_text(encoding="utf-8")) - for n in graph.get("nodes", []): - n.pop("community", None) - graph_path.write_text(json.dumps(graph), encoding="utf-8") - - r = _run(["export", "html"], tmp_path) - # Should NOT crash. It may print a warning and skip rendering, but exit - # code stays clean — same behaviour as the pre-fallback empty-communities - # path, just no longer silently failing on the common case. - assert r.returncode == 0, r.stderr - - -def test_graph_json_node_ids_are_portable_across_checkout_paths(tmp_path): - """#1789: the committed graph.json's node ids must be relative to the scan - root — not embed the absolute path — so the same repo yields identical ids - on any machine/checkout and leaks no local username/home.""" - def _build(root: Path): - (root / "pkg").mkdir(parents=True) - (root / "pkg" / "mod.py").write_text("def f(): return 1\n") - (root / "pkg" / "app.py").write_text("from pkg.mod import f\ndef g(): return f()\n") - r = _run(["extract", ".", "--code-only", "--no-cluster"], root) - assert r.returncode == 0, r.stderr - data = json.loads((root / "graphify-out" / "graph.json").read_text()) - return sorted(n["id"] for n in data["nodes"]) - - a = _build(tmp_path / "alice_home" / "proj") - b = _build(tmp_path / "bob_elsewhere" / "checkout" / "proj") - assert a == b, f"node ids differ across checkout paths: {a} vs {b}" - leak = {"alice_home", "bob_elsewhere", "checkout", "tmp", "private", "users", "home", "var"} - assert not any(part in leak for ident in a for part in ident.split("_")), \ - f"node id embeds an absolute-path component: {a}" + env = dict(os.environ, GRAPHIFY_OUT=str(configured)) + result = _run(project, "query", "OverrideNode", env=env) + assert result.returncode == 0, result.stderr + assert "OverrideNode" in result.stdout diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 21fd2ca3a..8a2d6c374 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,100 +1,67 @@ -import json -import sys -import networkx as nx -from pathlib import Path -from graphify.build import build_from_json -from graphify.cluster import cluster, cohesion_score, remap_communities_to_previous, score_all - -FIXTURES = Path(__file__).parent / "fixtures" - -def make_graph(): - return build_from_json(json.loads((FIXTURES / "extraction.json").read_text())) - -def test_cluster_returns_dict(): - G = make_graph() - communities = cluster(G) - assert isinstance(communities, dict) - -def test_cluster_covers_all_nodes(): - G = make_graph() - communities = cluster(G) - all_nodes = {n for nodes in communities.values() for n in nodes} - assert all_nodes == set(G.nodes) - -def test_cohesion_score_complete_graph(): - G = nx.complete_graph(4) - G = nx.relabel_nodes(G, {i: str(i) for i in G.nodes}) - score = cohesion_score(G, list(G.nodes)) - assert score == 1.0 - -def test_cohesion_score_single_node(): - G = nx.Graph() - G.add_node("a") - score = cohesion_score(G, ["a"]) - assert score == 1.0 - -def test_cohesion_score_disconnected(): - G = nx.Graph() - G.add_nodes_from(["a", "b", "c"]) - score = cohesion_score(G, ["a", "b", "c"]) - assert score == 0.0 - -def test_cohesion_score_range(): - G = make_graph() - communities = cluster(G) - for cid, nodes in communities.items(): - score = cohesion_score(G, nodes) - assert 0.0 <= score <= 1.0 - -def test_score_all_keys_match_communities(): - G = make_graph() - communities = cluster(G) - scores = score_all(G, communities) - assert set(scores.keys()) == set(communities.keys()) - +"""Native Leiden and community-state behavior.""" + +from graphify.cluster import ( + cluster, + cohesion_score, + community_member_sigs, + label_communities_by_hub, + remap_communities_to_previous, + score_all, +) +from tests.native_helpers import graph_from_payload + + +def _dense_groups(): + nodes = [{"id": name, "label": name.upper()} for name in "abcdef"] + edges = [ + {"source": left, "target": right, "relation": "related", "weight": 1.0} + for group in ("abc", "def") + for index, left in enumerate(group) + for right in group[index + 1:] + ] + return graph_from_payload(nodes, edges) + + +def test_native_leiden_clusters_dense_groups(): + communities = cluster(_dense_groups()) + assert {frozenset(group) for group in communities.values()} == { + frozenset("abc"), frozenset("def") + } -def test_cluster_does_not_write_to_stdout(capsys): - """Clustering should not emit ANSI escape codes or other output. - graspologic's leiden() can emit ANSI escape sequences that break - PowerShell 5.1's scroll buffer on Windows (issue #19). The output - suppression in _partition() should prevent any output from leaking. - """ - G = make_graph() - cluster(G) - captured = capsys.readouterr() - assert captured.out == "", f"cluster() wrote to stdout: {captured.out!r}" +def test_isolates_cohesion_and_score_keys(): + graph = graph_from_payload([{"id": "a"}, {"id": "b"}]) + communities = cluster(graph) + assert communities == {0: ["a"], 1: ["b"]} + assert cohesion_score(graph, ["a"]) == 1.0 + assert cohesion_score(graph, ["a", "b"]) == 0.0 + assert set(score_all(graph, communities)) == set(communities) -def test_cluster_does_not_write_to_stderr(capsys): - """Same as above but for stderr — ANSI codes can go to either stream.""" - G = make_graph() - cluster(G) - captured = capsys.readouterr() - # Allow logging output (starts with [graphify]) but no raw ANSI codes - for line in captured.err.splitlines(): - assert "\x1b" not in line, f"cluster() wrote ANSI to stderr: {line!r}" +def test_hub_labels_are_deterministic(): + graph = graph_from_payload( + [{"id": "hub", "label": "Hub()"}, {"id": "leaf", "label": "Leaf"}], + [{"source": "hub", "target": "leaf", "relation": "calls"}], + ) + assert label_communities_by_hub(graph, {3: ["leaf", "hub"]}) == {3: "Hub"} -def test_remap_communities_to_previous_reuses_old_ids(): - communities = { - 10: ["a", "b", "c"], - 11: ["d", "e"], - } +def test_remap_and_signatures_are_stable(): + communities = {10: ["a", "b", "c"], 11: ["d", "e"]} previous = {"a": 5, "b": 5, "c": 5, "d": 1, "e": 1} remapped = remap_communities_to_previous(communities, previous) - assert set(remapped.keys()) == {1, 5} - assert remapped[5] == ["a", "b", "c"] - assert remapped[1] == ["d", "e"] - - -def test_remap_communities_to_previous_assigns_deterministic_new_ids(): - communities = { - 7: ["x", "y", "z"], - 8: ["m"], + assert remapped == {5: ["a", "b", "c"], 1: ["d", "e"]} + assert community_member_sigs({0: ["a", "b"]}) == community_member_sigs({0: ["b", "a"]}) + + +def test_hub_exclusion_preserves_every_node(): + nodes = [{"id": "hub"}] + [{"id": f"n{i}"} for i in range(12)] + edges = [ + {"source": "hub", "target": f"n{i}", "relation": "uses"} + for i in range(12) + ] + graph = graph_from_payload(nodes, edges) + communities = cluster(graph, exclude_hubs_percentile=80) + assert {node for members in communities.values() for node in members} == { + node["id"] for node in nodes } - previous = {"a": 3} - remapped = remap_communities_to_previous(communities, previous) - assert list(remapped.keys()) == [0, 1] - assert remapped[0] == ["x", "y", "z"] - assert remapped[1] == ["m"] diff --git a/tests/test_community_hub_labels.py b/tests/test_community_hub_labels.py index 281a853d9..3b3b62266 100644 --- a/tests/test_community_hub_labels.py +++ b/tests/test_community_hub_labels.py @@ -4,20 +4,18 @@ instead of "Community 70", with no backend. Ties break by node id for run-to-run stability; a community with no members present in the graph falls back to "Community N". """ -import networkx as nx - from graphify.cluster import label_communities_by_hub +from tests.native_helpers import graph_from_payload def _g(node_labels, edges): - g = nx.Graph() - for nid, label in node_labels.items(): - if label is None: - g.add_node(nid) - else: - g.add_node(nid, label=label) - g.add_edges_from(edges) - return g + return graph_from_payload( + [ + {"id": nid, **({"label": label} if label is not None else {})} + for nid, label in node_labels.items() + ], + [{"source": source, "target": target} for source, target in edges], + ) def test_labels_by_highest_degree_hub(): @@ -50,9 +48,7 @@ def test_absent_members_fall_back_to_placeholder(): def test_node_without_label_attr_uses_id(): - g = nx.Graph() - g.add_nodes_from(["hub", "x", "y"]) - g.add_edges_from([("hub", "x"), ("hub", "y")]) # hub degree 2, no label attrs + g = _g({"hub": None, "x": None, "y": None}, [("hub", "x"), ("hub", "y")]) assert label_communities_by_hub(g, {0: ["hub", "x", "y"]})[0] == "hub" diff --git a/tests/test_confidence.py b/tests/test_confidence.py index 299548aca..9fc5c7a09 100644 --- a/tests/test_confidence.py +++ b/tests/test_confidence.py @@ -1,192 +1,12 @@ -"""Tests for confidence_score on edges.""" -import json -import tempfile -from pathlib import Path +from graphify.helix.access import first_edge_attributes +from tests.native_helpers import graph_from_payload -import networkx as nx -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.export import to_json -from graphify.report import generate - -FIXTURES = Path(__file__).parent / "fixtures" - - -def _make_extraction(**edge_overrides): - """Return a minimal extraction dict with one edge of each confidence type.""" - base = { - "nodes": [ - {"id": "n_a", "label": "A", "file_type": "code", "source_file": "a.py"}, - {"id": "n_b", "label": "B", "file_type": "code", "source_file": "b.py"}, - {"id": "n_c", "label": "C", "file_type": "document", "source_file": "c.md"}, - {"id": "n_d", "label": "D", "file_type": "document", "source_file": "d.md"}, - ], - "edges": [ - {"source": "n_a", "target": "n_b", "relation": "calls", "confidence": "EXTRACTED", - "confidence_score": 1.0, "source_file": "a.py", "weight": 1.0}, - {"source": "n_b", "target": "n_c", "relation": "implements", "confidence": "INFERRED", - "confidence_score": 0.75, "source_file": "b.py", "weight": 0.8}, - {"source": "n_c", "target": "n_d", "relation": "references", "confidence": "AMBIGUOUS", - "confidence_score": 0.2, "source_file": "c.md", "weight": 0.5}, - ], - "input_tokens": 100, - "output_tokens": 50, - } - return base - - -def test_extracted_edges_have_score_1(): - """EXTRACTED edges must have confidence_score == 1.0.""" - G = build_from_json(_make_extraction()) - for u, v, d in G.edges(data=True): - if d.get("confidence") == "EXTRACTED": - assert d.get("confidence_score") == 1.0, ( - f"EXTRACTED edge ({u},{v}) should have confidence_score=1.0, got {d.get('confidence_score')}" - ) - - -def test_inferred_edges_score_in_range(): - """INFERRED edges must have confidence_score between 0.0 and 1.0.""" - G = build_from_json(_make_extraction()) - found = False - for u, v, d in G.edges(data=True): - if d.get("confidence") == "INFERRED": - found = True - score = d.get("confidence_score") - assert score is not None, f"INFERRED edge ({u},{v}) missing confidence_score" - assert 0.0 <= score <= 1.0, ( - f"INFERRED edge ({u},{v}) confidence_score={score} out of range [0,1]" - ) - assert found, "No INFERRED edges found in test fixture" - - -def test_ambiguous_edges_score_at_most_04(): - """AMBIGUOUS edges must have confidence_score <= 0.4.""" - G = build_from_json(_make_extraction()) - found = False - for u, v, d in G.edges(data=True): - if d.get("confidence") == "AMBIGUOUS": - found = True - score = d.get("confidence_score") - assert score is not None, f"AMBIGUOUS edge ({u},{v}) missing confidence_score" - assert score <= 0.4, ( - f"AMBIGUOUS edge ({u},{v}) confidence_score={score} should be <= 0.4" - ) - assert found, "No AMBIGUOUS edges found in test fixture" - - -def test_confidence_score_round_trip(): - """confidence_score survives build_from_json → to_json → JSON parse round-trip.""" - extraction = _make_extraction() - G = build_from_json(extraction) - communities = cluster(G) - - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - data = json.loads(out.read_text()) - - # to_json uses node_link_data which puts edges in "links" - links = data.get("links", []) - assert links, "No links found in exported graph.json" - for link in links: - assert "confidence_score" in link, f"Link missing confidence_score: {link}" - score = link["confidence_score"] - assert isinstance(score, float), f"confidence_score should be float, got {type(score)}" - assert 0.0 <= score <= 1.0, f"confidence_score={score} out of range" - - -def test_to_json_defaults_missing_confidence_score(): - """Edges lacking confidence_score get sensible defaults in to_json.""" - extraction = { - "nodes": [ - {"id": "n_x", "label": "X", "file_type": "code", "source_file": "x.py"}, - {"id": "n_y", "label": "Y", "file_type": "code", "source_file": "y.py"}, - {"id": "n_z", "label": "Z", "file_type": "code", "source_file": "z.py"}, - ], - "edges": [ - # No confidence_score field on any of these - {"source": "n_x", "target": "n_y", "relation": "calls", - "confidence": "EXTRACTED", "source_file": "x.py", "weight": 1.0}, - {"source": "n_y", "target": "n_z", "relation": "depends_on", - "confidence": "INFERRED", "source_file": "y.py", "weight": 1.0}, - ], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(extraction) - communities = cluster(G) - - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - data = json.loads(out.read_text()) - - links_by_conf = {} - for link in data.get("links", []): - conf = link.get("confidence", "EXTRACTED") - links_by_conf[conf] = link.get("confidence_score") - - assert links_by_conf.get("EXTRACTED") == 1.0, "EXTRACTED default should be 1.0" - assert links_by_conf.get("INFERRED") == 0.5, "INFERRED default should be 0.5" - - -def test_report_shows_avg_confidence_for_inferred(): - """Report summary line should include avg confidence for INFERRED edges.""" - extraction = _make_extraction() - G = build_from_json(extraction) - communities = cluster(G) - cohesion = score_all(G, communities) - labels = {cid: f"Community {cid}" for cid in communities} - gods = god_nodes(G) - surprises = surprising_connections(G) - detection = {"total_files": 2, "total_words": 5000, "needs_graph": True, "warning": None} - tokens = {"input": 100, "output": 50} - - report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, ".") - assert "avg confidence" in report, "Report should show avg confidence for INFERRED edges" - # The fixture has one INFERRED edge with score 0.75, so avg should be 0.75 - assert "0.75" in report, f"Expected avg confidence 0.75 in report" - - -def test_report_inferred_tag_with_score(): - """Surprising connections section shows confidence score next to INFERRED edges.""" - # Build a graph where surprising_connections will find an INFERRED cross-file edge - extraction = { - "nodes": [ - {"id": "n_p", "label": "Parser", "file_type": "code", "source_file": "parser.py"}, - {"id": "n_q", "label": "Renderer", "file_type": "code", "source_file": "renderer.py"}, - ], - "edges": [ - {"source": "n_p", "target": "n_q", "relation": "feeds", - "confidence": "INFERRED", "confidence_score": 0.82, - "source_file": "parser.py", "weight": 1.0}, - ], - "input_tokens": 0, - "output_tokens": 0, - } - G = build_from_json(extraction) - - # Manually construct a surprise entry the way analyze.surprising_connections would - surprise = { - "source": "Parser", - "target": "Renderer", - "relation": "feeds", - "confidence": "INFERRED", - "confidence_score": 0.82, - "source_files": ["parser.py", "renderer.py"], - "note": "", - } - communities = cluster(G) - cohesion = score_all(G, communities) - labels = {cid: f"Community {cid}" for cid in communities} - gods = god_nodes(G) - detection = {"total_files": 2, "total_words": 1000, "needs_graph": True, "warning": None} - tokens = {"input": 0, "output": 0} - - report = generate(G, communities, cohesion, labels, gods, [surprise], detection, tokens, ".") - assert "INFERRED 0.82" in report, ( - f"Report should show 'INFERRED 0.82' in surprising connections section. Got:\n{report}" +def test_confidence_and_weight_round_trip_natively(): + graph = graph_from_payload( + [{"id": "a"}, {"id": "b"}], + [{"source": "a", "target": "b", "relation": "calls", "confidence": "AMBIGUOUS", "weight": 0.2}], ) + attrs = first_edge_attributes(graph, "a", "b") + assert attrs["confidence"] == "AMBIGUOUS" + assert attrs["weight"] == 0.2 diff --git a/tests/test_corrupt_graph_json.py b/tests/test_corrupt_graph_json.py deleted file mode 100644 index 7f2258c38..000000000 --- a/tests/test_corrupt_graph_json.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Corrupt graph.json produces an actionable error, not a raw traceback (#1536/#1537). - -Three load paths call json.loads on graph.json — build_merge (`--update`), -affected.load_graph (`graphify prs`), and diagnostics._read_json_file -(`graphify diagnose`). A truncated / invalid file (incomplete write, power loss, -manual edit) must raise a clear RuntimeError with recovery guidance at each. -""" -from __future__ import annotations - -import pytest - -from graphify.build import build_merge -from graphify.affected import load_graph -from graphify.diagnostics import _read_json_file - -_CORRUPT = '{"nodes": [{"id": "a", "labe' # truncated mid-object - - -def _corrupt(tmp_path): - p = tmp_path / "graph.json" - p.write_text(_CORRUPT, encoding="utf-8") - return p - - -def test_build_merge_corrupt_graph_raises_runtimeerror(tmp_path): - p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot read .*incremental merge|rebuild"): - build_merge([], graph_path=p, dedup=False) - - -def test_affected_load_graph_corrupt_raises_runtimeerror(tmp_path): - p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot read graph file|regenerate"): - load_graph(p) - - -def test_diagnostics_read_corrupt_raises_runtimeerror(tmp_path): - p = _corrupt(tmp_path) - with pytest.raises(RuntimeError, match=r"Cannot parse|corrupted"): - _read_json_file(p) - - -def test_valid_graph_still_loads(tmp_path): - """Happy path unchanged: a well-formed graph.json loads without raising.""" - p = tmp_path / "graph.json" - p.write_text( - '{"nodes": [{"id": "a", "label": "a", "file_type": "code"}], "edges": []}', - encoding="utf-8", - ) - # none of these should raise - load_graph(p) - _read_json_file(p) - build_merge([], graph_path=p, dedup=False) diff --git a/tests/test_cpp_objc_cross_file_calls.py b/tests/test_cpp_objc_cross_file_calls.py index 051a24373..21358c214 100644 --- a/tests/test_cpp_objc_cross_file_calls.py +++ b/tests/test_cpp_objc_cross_file_calls.py @@ -9,7 +9,7 @@ from pathlib import Path -from graphify.build import build_from_json +from graphify.build import build_from_extraction from graphify.extract import extract @@ -165,7 +165,7 @@ def test_cpp_godnode_guard_ambiguous_and_unknown_receiver(tmp_path: Path): def test_cpp_resolved_call_survives_build(tmp_path: Path): - # The receiver-typed call targets the header-declared method node; build_from_json + # The receiver-typed call targets the header-declared method node; build_from_extraction # must keep it. The cross-language INFERRED-call guard treats C/C++ as one family, # so a `.cpp` -> `.h`-declared-method edge is not pruned (#1547). base = tmp_path / "src" @@ -174,10 +174,11 @@ def test_cpp_resolved_call_survives_build(tmp_path: Path): _write(base / "Main.cpp", '#include "Foo.h"\nint main() { Foo f; f.bar(); }\n') result = extract(sorted(base.glob("*")), cache_root=tmp_path / "cache") - g = build_from_json(result) + g = build_from_extraction(result) cross = [ - d for _, _, d in g.edges(data=True) - if d.get("relation") == "calls" and d.get("confidence") == "INFERRED" + edge.attributes for edge in g.edges + if edge.attributes.get("relation") == "calls" + and edge.attributes.get("confidence") == "INFERRED" ] assert len(cross) >= 1 @@ -246,7 +247,7 @@ def test_objc_godnode_guard_ambiguous_selector(tmp_path: Path): def test_objc_resolved_calls_survive_build(tmp_path: Path): # The cross-file ObjC call must land on a real definition node so - # build_from_json keeps it (no dangling target pruned). + # build_from_extraction keeps it (no dangling target pruned). base = tmp_path / "src" _write(base / "Foo.h", "@interface Foo : NSObject\n- (void)doThing;\n@end\n") _write(base / "Foo.m", '#import "Foo.h"\n@implementation Foo\n- (void)doThing {}\n@end\n') @@ -255,9 +256,10 @@ def test_objc_resolved_calls_survive_build(tmp_path: Path): '- (void)go {\n Foo *f = [[Foo alloc] init];\n [f doThing];\n}\n@end\n') result = extract(sorted(base.glob("*")), cache_root=tmp_path / "cache") - g = build_from_json(result) + g = build_from_extraction(result) cross = [ - d for _, _, d in g.edges(data=True) - if d.get("relation") == "calls" and d.get("confidence") == "INFERRED" + edge.attributes for edge in g.edges + if edge.attributes.get("relation") == "calls" + and edge.attributes.get("confidence") == "INFERRED" ] assert len(cross) >= 1 diff --git a/tests/test_cross_extension_reexport_self_cycle.py b/tests/test_cross_extension_reexport_self_cycle.py index 7b66699ac..06fe8cb95 100644 --- a/tests/test_cross_extension_reexport_self_cycle.py +++ b/tests/test_cross_extension_reexport_self_cycle.py @@ -20,8 +20,9 @@ from pathlib import Path from graphify.build import build -from graphify.export import to_json from graphify.extract import extract +from graphify.helix.model import edge_attributes +from tests.native_helpers import graph_from_build, graph_from_payload def _write(path: Path, text: str) -> Path: @@ -97,8 +98,6 @@ def test_cross_ext_reexport_target_is_the_sibling_node(tmp_path: Path): def test_cross_ext_reexport_no_phantom_import_cycle(tmp_path: Path): - import networkx as nx - from graphify.analyze import find_import_cycles mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") @@ -106,15 +105,12 @@ def test_cross_ext_reexport_no_phantom_import_cycle(tmp_path: Path): result = extract([mjs, ts], cache_root=tmp_path) - graph = nx.DiGraph() - for node in result["nodes"]: - graph.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) - for edge in result["edges"]: - graph.add_edge( - edge["source"], - edge["target"], - **{k: v for k, v in edge.items() if k not in ("source", "target")}, - ) + node_ids = {node["id"] for node in result["nodes"]} + edges = [ + edge for edge in result["edges"] + if edge.get("source") in node_ids and edge.get("target") in node_ids + ] + graph = graph_from_payload(result["nodes"], edges, kind="multidigraph") assert find_import_cycles(graph) == [], ( "cross-extension re-export must not manufacture a file-level import cycle" ) @@ -169,8 +165,8 @@ def test_same_basename_three_colliding_siblings_reexport_selects_named_variant( # into graph.json breaks determinism across checkout locations and the # cross-machine merge/global-graph portability the codebase engineered for. The # following lock that the hint is stripped after disambiguation and never -# reaches graph.json — on the raw-dump extract path AND the build path — and -# that a persisted absolute hint from a pre-fix graph is dropped on the next +# reaches the native store — on the raw extract path AND the build path — and +# that a stale absolute hint from a pre-fix extraction is dropped on the next # build rather than carried forward. # --------------------------------------------------------------------------- # @@ -205,30 +201,23 @@ def test_target_file_hint_stripped_even_without_a_collision(tmp_path: Path): ) -def test_graph_json_has_no_target_file_and_no_absolute_path(tmp_path: Path): +def test_native_store_has_no_target_file_and_no_absolute_path(tmp_path: Path): mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') result = extract([mjs, ts], cache_root=tmp_path) graph = build([result], root=tmp_path) - out = tmp_path / "graph.json" - to_json(graph, {}, str(out), force=True) - - raw = out.read_text(encoding="utf-8") - data = json.loads(raw) - leaked = [link for link in data["links"] if "target_file" in link] - assert not leaked, f"absolute target_file persisted into graph.json links: {leaked}" - assert str(tmp_path.resolve()) not in raw, ( - "graph.json leaked an absolute checkout path (source_file is relativized, " - "but the target_file hint was serialized verbatim)" + native = graph_from_build(graph) + attrs = [edge_attributes(edge) for edge in native.edges()] + leaked = [edge for edge in attrs if "target_file" in edge] + assert not leaked, f"absolute target_file persisted into native edges: {leaked}" + assert str(tmp_path.resolve()) not in json.dumps(attrs, ensure_ascii=False), ( + "native edges leaked an absolute checkout path" ) -def test_graph_json_is_checkout_location_independent(tmp_path: Path): - """Building the byte-identical repo at two different absolute locations must - yield identical graph.json edges. A leaked absolute target_file differs by - its checkout prefix and would defeat the cross-machine merge/global-graph - portability the codebase is built around.""" +def test_native_edges_are_checkout_location_independent(tmp_path: Path): + """Byte-identical repos at different locations produce identical native edges.""" def _links_built_at(dirname: str) -> list[dict]: d = tmp_path / dirname @@ -237,9 +226,11 @@ def _links_built_at(dirname: str) -> list[dict]: ts = _write(d / "foo.ts", 'export { N } from "./foo.mjs";\n') result = extract([mjs, ts], cache_root=d) graph = build([result], root=d) - out = d / "graph.json" - to_json(graph, {}, str(out), force=True) - links = json.loads(out.read_text(encoding="utf-8"))["links"] + native = graph_from_build(graph) + links = [ + {"source": edge.source, "target": edge.target, **edge_attributes(edge)} + for edge in native.edges() + ] return sorted( links, key=lambda link: ( @@ -250,16 +241,13 @@ def _links_built_at(dirname: str) -> list[dict]: ) assert _links_built_at("loc_a") == _links_built_at("loc_bbbb_longer"), ( - "graph.json edges differ across checkout locations — an absolute path leaked" + "native edges differ across checkout locations — an absolute path leaked" ) def test_build_drops_persisted_target_file_from_a_pre_fix_graph(tmp_path: Path): - # A graph.json written by a pre-fix build carries an absolute target_file on - # its import edges. On the next (incremental) build those base edges are - # re-serialized through build(), which does NOT re-run disambiguation — so - # the serializer itself must drop the persisted absolute path rather than - # carry a foreign checkout prefix forward into the updated graph. + # A stale pre-fix extraction carries an absolute target_file on its import + # edges. The build DTO must drop it before native persistence. legacy_chunk = { "nodes": [ {"id": "foo_ts_foo", "label": "foo.ts", @@ -283,10 +271,11 @@ def test_build_drops_persisted_target_file_from_a_pre_fix_graph(tmp_path: Path): graph = build([legacy_chunk], root=tmp_path) - assert graph.number_of_edges() == 1, "the base import edge should survive the merge" - for _src, _tgt, data in graph.edges(data=True): - assert "target_file" not in data, ( - f"build() carried a persisted absolute target_file into the graph: {data}" + assert graph.edge_count == 1, "the base import edge should survive the merge" + for edge in graph.edges: + assert "target_file" not in edge.attributes, ( + "build() carried a persisted absolute target_file into the graph: " + f"{edge.attributes}" ) diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 57fe6c192..b112953d0 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -134,7 +134,7 @@ def test_build_calls_dedup(): "edges": [], } G = build([chunk1, chunk2]) - assert G.number_of_nodes() == 1 + assert G.node_count == 1 # --- #878: fuzzy dedup false merges on short/variant labels --- @@ -183,7 +183,6 @@ def test_prefix_extension_symbols_not_merged(): """Distinct symbols whose name is a strict prefix-extension of another must not be merged (#1201). getActiveSession / getActiveSessions score ~98.82 JW but are different functions; parseConfig / parseConfigFile likewise.""" - import networkx as nx from graphify.dedup import deduplicate_entities pairs = [ diff --git a/tests/test_devin.py b/tests/test_devin.py index a3bca5d05..8a75b8a8d 100644 --- a/tests/test_devin.py +++ b/tests/test_devin.py @@ -236,19 +236,12 @@ def test_devin_skill_file_exists_in_package(): assert skill.exists(), "skill-devin.md missing from package" -def test_devin_skill_file_uses_python_c_syntax(): - """Devin skill must use inline python -c syntax (cross-platform, no bash heredocs). - - All mature graphify skills use the interpreter-detection pattern - ``$(cat graphify-out/.graphify_python) -c "..."`` rather than bare - ``python -c "..."`` so they work in pipx / venv environments. - """ +def test_devin_skill_file_uses_installed_native_cli(): + """Devin delegates builds and queries to the installed Helix-native CLI.""" import graphify skill = (Path(graphify.__file__).parent / "skill-devin.md").read_text() - assert '.graphify_python) -c "' in skill, ( - "skill-devin.md must use the interpreter-detection pattern " - "'$(cat graphify-out/.graphify_python) -c \"...\"'" - ) + assert "graphify extract INPUT_PATH" in skill + assert "graphify-out/graph.helix" in skill assert "#!/bin/bash" not in skill diff --git a/tests/test_dotnet.py b/tests/test_dotnet.py index fac4fd0e6..06c2efdbc 100644 --- a/tests/test_dotnet.py +++ b/tests/test_dotnet.py @@ -48,7 +48,7 @@ def test_sln_project_dependency(): def test_sln_solution_folder_ids_are_relative(tmp_path): """Solution folders are virtual groupings, not files. Their node ids must be derived from the folder name only — never the resolved absolute scan path, - which would leak the local username into a committed graph.json (#1789).""" + which would leak the local username into a committed graph store (#1789).""" sln = tmp_path / "App.sln" sln.write_text( 'Microsoft Visual Studio Solution File, Format Version 12.00\n' diff --git a/tests/test_explain_cli.py b/tests/test_explain_cli.py index 7759f30e3..43c973ca6 100644 --- a/tests/test_explain_cli.py +++ b/tests/test_explain_cli.py @@ -1,125 +1,73 @@ -"""Regression tests for `graphify explain` arrow direction (#853).""" -from __future__ import annotations -import json -import graphify.__main__ as mainmod - +"""Native ``graphify explain`` direction and learning-state tests.""" -def _write_graph(tmp_path): - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ - {"id": "validate", "label": "validateSanitySession()", - "source_file": "server/sanity-validate-session.ts", "community": 0}, - {"id": "create_patch", "label": "createPatchHandler()", - "source_file": "server/create-patch-handler.ts", "community": 0}, - {"id": "create_edit", "label": "createEditHandler()", - "source_file": "server/create-edit-handler.ts", "community": 0}, - {"id": "stable_stringify", "label": "stableStringify()", - "source_file": "shared/stringify.ts", "community": 0}, +import graphify.__main__ as mainmod +from graphify.helix.state import new_state +from tests.native_helpers import make_loaded + + +def _store(tmp_path, *, learning=None): + state = new_state(learning=learning or {}) + return make_loaded( + tmp_path, + nodes=[ + {"id": "validate", "label": "validateSanitySession()", "source_file": "server/validate.ts", "source_location": "L1", "file_type": "code"}, + {"id": "patch", "label": "createPatchHandler()", "source_file": "server/patch.ts", "file_type": "code"}, + {"id": "edit", "label": "createEditHandler()", "source_file": "server/edit.ts", "file_type": "code"}, + {"id": "stringify", "label": "stableStringify()", "source_file": "shared/stringify.ts", "file_type": "code"}, ], - "links": [ - {"source": "create_patch", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "create_edit", "target": "validate", - "relation": "calls", "confidence": "EXTRACTED"}, - {"source": "validate", "target": "stable_stringify", - "relation": "calls", "confidence": "EXTRACTED"}, + edges=[ + {"source": "patch", "target": "validate", "relation": "calls"}, + {"source": "edit", "target": "validate", "relation": "calls"}, + {"source": "validate", "target": "stringify", "relation": "calls"}, ], - } - p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) - return p + state=state, + ) -def _run(monkeypatch, graph_path, label, capsys): +def _run(monkeypatch, capsys, store, query): monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr(mainmod.sys, "argv", - ["graphify", "explain", label, "--graph", str(graph_path)]) + monkeypatch.setattr(mainmod.sys, "argv", ["graphify", "explain", query, "--store", str(store)]) mainmod.main() return capsys.readouterr().out -def test_callee_shows_callers_as_inbound(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) - out = _run(monkeypatch, p, "validateSanitySession", capsys) +def test_callee_shows_callers_inbound_and_callee_outbound(monkeypatch, tmp_path, capsys): + store = _store(tmp_path).store_path + out = _run(monkeypatch, capsys, store, "validateSanitySession") assert "<-- createPatchHandler() [calls]" in out assert "<-- createEditHandler() [calls]" in out assert "--> stableStringify() [calls]" in out - assert "--> createPatchHandler() [calls]" not in out - assert "--> createEditHandler() [calls]" not in out -def test_caller_shows_callee_as_outbound(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) - out = _run(monkeypatch, p, "createPatchHandler", capsys) +def test_caller_shows_callee_outbound(monkeypatch, tmp_path, capsys): + store = _store(tmp_path).store_path + out = _run(monkeypatch, capsys, store, "createPatchHandler") assert "--> validateSanitySession() [calls]" in out - assert "<-- " not in out -def test_explain_source_file_path_prefers_file_level_node(monkeypatch, tmp_path, capsys): - source_file = "app/api/example/route.ts" - graph_data = { - "directed": False, "multigraph": False, "graph": {}, - "nodes": [ - {"id": "example_route_get", "label": "GET()", - "source_file": source_file, "source_location": "L42", "community": 0}, - {"id": "example_route", "label": "route.ts", - "source_file": source_file, "source_location": "L1", "community": 0}, - ], - "links": [ - {"source": "example_route", "target": "example_route_get", - "relation": "contains", "confidence": "EXTRACTED"}, +def test_source_path_exact_match_prefers_file_node(monkeypatch, tmp_path, capsys): + loaded = make_loaded( + tmp_path, + nodes=[ + {"id": "route_get", "label": "GET()", "source_file": "app/route.ts", "source_location": "L42"}, + {"id": "route", "label": "route.ts", "source_file": "app/route.ts", "source_location": "L1"}, ], - } - p = tmp_path / "graph.json" - p.write_text(json.dumps(graph_data)) - - out = _run(monkeypatch, p, source_file, capsys) - - assert "Node: route.ts" in out - assert "ID: example_route" in out - assert f"Source: {source_file} L1" in out - assert "Node: GET()" not in out - - -# --- work-memory overlay Lesson line ------------------------------------------ - -def _write_sidecar(tmp_path, nodes): - (tmp_path / ".graphify_learning.json").write_text( - json.dumps({"version": 1, "generated_at": "2026-06-01T00:00:00+00:00", - "nodes": nodes}), - encoding="utf-8", + edges=[{"source": "route", "target": "route_get", "relation": "contains"}], ) + out = _run(monkeypatch, capsys, loaded.store_path, "app/route.ts") + assert "Node: route.ts" in out and "ID: route" in out -def test_explain_shows_preferred_lesson_line(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) - _write_sidecar(tmp_path, { - "validate": {"status": "preferred", "score": 2.4, "uses": 3, - "label": "validateSanitySession()", "source_file": "", - "code_fingerprint": "", "provenance": []}, - }) - out = _run(monkeypatch, p, "validateSanitySession", capsys) +def test_learning_state_is_rendered(monkeypatch, tmp_path, capsys): + learning = { + "version": 1, + "nodes": {"validate": {"status": "preferred", "score": 2.4, "uses": 3}}, + } + store = _store(tmp_path, learning=learning).store_path + out = _run(monkeypatch, capsys, store, "validate") assert "Lesson: preferred source (start here) — 3 useful, score=2.4" in out - assert "code changed" not in out - - -def test_explain_shows_contested_and_stale_lesson(monkeypatch, tmp_path, capsys): - p = _write_graph(tmp_path) - # source_file points at a path that does not exist -> loader marks it stale. - _write_sidecar(tmp_path, { - "validate": {"status": "contested", "score": -0.1, "uses": 2, "neg": 1, - "verdict": "dead end", "label": "validateSanitySession()", - "source_file": "server/sanity-validate-session.ts", - "code_fingerprint": "deadbeef", "provenance": []}, - }) - out = _run(monkeypatch, p, "validateSanitySession", capsys) - assert "Lesson: contested (useful 2 / dead-end 1)" in out - assert "[code changed since — re-verify]" in out -def test_explain_no_lesson_line_for_unannotated_node(monkeypatch, tmp_path, capsys): - """No sidecar => no Lesson line; output identical to pre-feature.""" - p = _write_graph(tmp_path) - out = _run(monkeypatch, p, "validateSanitySession", capsys) +def test_unannotated_node_has_no_lesson(monkeypatch, tmp_path, capsys): + out = _run(monkeypatch, capsys, _store(tmp_path).store_path, "validate") assert "Lesson:" not in out diff --git a/tests/test_export.py b/tests/test_export.py index 28a4707a7..cec65a665 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,793 +1,61 @@ import json -import math -import re -import tempfile -from pathlib import Path -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import to_json, to_cypher, to_graphml, to_html, to_canvas, to_obsidian +import xml.etree.ElementTree as ET -FIXTURES = Path(__file__).parent / "fixtures" +from graphify.export import to_canvas, to_cypher, to_graphml, to_html, to_obsidian +from tests.native_helpers import graph_from_payload -def make_graph(): - return build_from_json(json.loads((FIXTURES / "extraction.json").read_text())) -def test_to_json_creates_file(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - assert out.exists() - -def test_to_json_valid_json(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - data = json.loads(out.read_text()) - assert "nodes" in data - assert "links" in data - -def test_to_json_nodes_have_community(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.json" - to_json(G, communities, str(out)) - data = json.loads(out.read_text()) - for node in data["nodes"]: - assert "community" in node - -def test_to_cypher_creates_file(): - G = make_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "cypher.txt" - to_cypher(G, str(out)) - assert out.exists() - -def test_to_cypher_contains_merge_statements(): - G = make_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "cypher.txt" - to_cypher(G, str(out)) - content = out.read_text() - assert "MERGE" in content - -def test_to_graphml_creates_file(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.graphml" - to_graphml(G, communities, str(out)) - assert out.exists() - -def test_to_graphml_valid_xml(): - G = make_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.graphml" - to_graphml(G, communities, str(out)) - content = out.read_text() - assert " "" so a node/edge with a null field still exports (#1502).""" - G = make_graph() - communities = cluster(G) - # Inject a None-valued attribute on one node and one edge. - a_node = next(iter(G.nodes())) - G.nodes[a_node]["nullable_field"] = None - if G.number_of_edges(): - u, v = next(iter(G.edges())) - G.edges[u, v]["nullable_field"] = None - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.graphml" - to_graphml(G, communities, str(out)) # must not raise - content = out.read_text() - assert " list: - """Extract the RAW_NODES JSON array embedded in the generated HTML.""" - m = re.search(r"const RAW_NODES = (\[.*?\]);", content, re.DOTALL) - assert m, "RAW_NODES not found in HTML" - return json.loads(m.group(1).replace("<\\/", "= G.number_of_nodes() - assert len(data["edges"]) >= 1 - assert out.stat().st_size > 32 - - -def test_to_canvas_node_grid_matches_box_columns(): - """#1452: a community's node cards are laid out in the same ceil(sqrt(n))-column - grid the group box is sized for. Previously the box width assumed sqrt(n) - columns while the placement loop hardcoded 3, so any community bigger than ~9 - rendered as a cramped 3-wide strip filling only part of an over-wide box. - Covers a perfect square (25 -> 5x5) and a non-square count (10 -> 4 cols, a - partial last row) so both the column count and the row count are pinned.""" - for n in (10, 25): - G = build_from_json({ - "nodes": [ - {"id": f"n{i}", "label": f"sym_{i:02d}", "file_type": "code", "source_file": "a.py"} - for i in range(n) - ], - "edges": [], - }) - communities = {0: [f"n{i}" for i in range(n)]} - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.canvas" - to_canvas(G, communities, str(out)) - data = json.loads(out.read_text()) - - group = next(g for g in data["nodes"] if g.get("type") == "group") - cards = [c for c in data["nodes"] if c.get("type") == "file"] - assert len(cards) == n, f"n={n}" - - # Cards occupy the ceil(sqrt(n))-column / ceil(n/cols)-row grid the box is - # sized for — not the old fixed 3 columns, which spread cards across far - # more rows (the load-bearing checks: distinct column/row positions). - expected_cols = math.ceil(math.sqrt(n)) - expected_rows = math.ceil(n / expected_cols) - distinct_x = len({c["x"] for c in cards}) - distinct_y = len({c["y"] for c in cards}) - assert distinct_x == expected_cols, f"n={n}: expected {expected_cols} cols, got {distinct_x}" - assert distinct_y == expected_rows, f"n={n}: expected {expected_rows} rows, got {distinct_y}" - - # And every card sits fully inside its group box on both axes. - gx, gy, gw, gh = group["x"], group["y"], group["width"], group["height"] - for c in cards: - assert gx <= c["x"] and c["x"] + c["width"] <= gx + gw, (n, c) - assert gy <= c["y"] and c["y"] + c["height"] <= gy + gh, (n, c) - - -# ── Issue #1409: punctuation-only Obsidian/Canvas filenames ─────────────────── - -def _punct_graph(label: str): - """A 2-node graph where one node's label is all-punctuation (e.g. a `@/*` - tsconfig paths key) and the other is a normal symbol.""" - return build_from_json({ - "nodes": [ - {"id": "n1", "label": label, "file_type": "code", "source_file": "tsconfig.json"}, - {"id": "n2", "label": "AuthHandler", "file_type": "code", "source_file": "auth.ts"}, - ], - "edges": [], - }) - - -def test_to_obsidian_never_emits_punctuation_only_filenames(): - """#1409: an all-punctuation label (e.g. `@/*`) must not produce a `@.md`-style - filename — valid on disk but empty once a downstream tool re-slugs on word chars - (crashes `qmd update`). It falls back to `unnamed`.""" - G = _punct_graph("@/*") - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp) - stems = [p.stem for p in Path(tmp).rglob("*.md")] - assert stems, "to_obsidian wrote no notes" - bad = [s for s in stems if not re.search(r"\w", s, flags=re.UNICODE)] - assert not bad, f"punctuation-only filenames emitted: {bad}" - assert any(s == "unnamed" or s.startswith("unnamed") for s in stems), stems - - -def test_to_canvas_never_emits_punctuation_only_filenames(): - """#1409: same guard on the canvas exporter's file-node names.""" - G = _punct_graph("@") - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.canvas" - to_canvas(G, communities, str(out)) - data = json.loads(out.read_text()) - file_nodes = [n for n in data["nodes"] if n.get("type") == "file"] - assert file_nodes, "canvas has no file nodes" - bad = [n["file"] for n in file_nodes if not re.search(r"\w", Path(n["file"]).stem, flags=re.UNICODE)] - assert not bad, f"punctuation-only canvas filenames: {bad}" - - -# ── Existing-vault safety: graphify must not clobber user notes / .obsidian (#1506) ── - -def _two_node_graph(): - import networkx as nx - G = nx.Graph() - G.add_node("n1", label="Database", community=0, source_file="app/db.py", type="code") - G.add_node("n2", label="Server", community=0, source_file="app/srv.py", type="code") - G.add_edge("n1", "n2") - return G, {0: ["n1", "n2"]} - - -def test_to_obsidian_preserves_existing_user_notes_and_obsidian_config(): - """#1506: exporting into an existing vault must not overwrite a user's note that - collides with a graphify node name, nor their .obsidian/ graph settings.""" - G, communities = _two_node_graph() - with tempfile.TemporaryDirectory() as tmp: - vault = Path(tmp) - (vault / "Database.md").write_text("# MY NOTES\nkeep me\n", encoding="utf-8") - (vault / ".obsidian").mkdir() - (vault / ".obsidian" / "graph.json").write_text('{"USER":"settings"}', encoding="utf-8") - to_obsidian(G, communities, str(vault), community_labels={0: "Backend"}) - # user content untouched - assert "MY NOTES" in (vault / "Database.md").read_text() - assert json.loads((vault / ".obsidian" / "graph.json").read_text()) == {"USER": "settings"} - # non-colliding graphify note still written - assert (vault / "Server.md").exists() - - -def test_to_obsidian_empty_dir_writes_full_vault(): - """No regression: a fresh/empty dir still gets every note + .obsidian/graph.json.""" - G, communities = _two_node_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "obsidian" - n = to_obsidian(G, communities, str(out), community_labels={0: "Backend"}) - assert (out / "Database.md").exists() and (out / "Server.md").exists() - assert (out / ".obsidian" / "graph.json").exists() - assert n == 3 # 2 nodes + 1 community note - - -def test_to_obsidian_rerun_updates_own_notes_but_not_user_files(): - """A re-run overwrites graphify's own prior notes (via the manifest) but leaves a - user-added note in the same dir alone.""" - G, communities = _two_node_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "obsidian" - to_obsidian(G, communities, str(out), community_labels={0: "Backend"}) - (out / "UserNote.md").write_text("mine\n", encoding="utf-8") - to_obsidian(G, communities, str(out), community_labels={0: "Backend2"}) - assert (out / "Database.md").exists() # graphify re-wrote its own - assert (out / "UserNote.md").read_text().strip() == "mine" # user's untouched - - -def _four_node_two_community_graph(): - import networkx as nx - G = nx.Graph() - G.add_node("n1", label="Database", community=0, source_file="app/db.py", type="code") - G.add_node("n2", label="Server", community=0, source_file="app/srv.py", type="code") - G.add_node("n3", label="Cache", community=1, source_file="infra/cache.py", type="code") - G.add_node("n4", label="Queue", community=1, source_file="infra/queue.py", type="code") - G.add_edge("n1", "n2") - G.add_edge("n3", "n4") - return G, {0: ["n1", "n2"], 1: ["n3", "n4"]} - - -def test_to_obsidian_rerun_prunes_removed_nodes(): - """#1896: re-exporting into the same vault must delete graphify's own notes for - nodes (and communities) that dropped out of the graph, so the vault mirrors the - current graph rather than old-union-new. User files are never touched.""" - G4, comm4 = _four_node_two_community_graph() - G2, comm2 = _two_node_graph() - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "obsidian" - to_obsidian(G4, comm4, str(out), community_labels={0: "Backend", 1: "Infra"}) - assert (out / "Cache.md").exists() and (out / "_COMMUNITY_Infra.md").exists() - (out / "MyOwnNote.md").write_text("mine\n", encoding="utf-8") - to_obsidian(G2, comm2, str(out), community_labels={0: "Backend"}) - # notes for removed nodes and the stale community overview are pruned - assert not (out / "Cache.md").exists() - assert not (out / "Queue.md").exists() - assert not (out / "_COMMUNITY_Infra.md").exists() - # surviving graphify notes and the user's own note remain - assert (out / "Database.md").exists() and (out / "Server.md").exists() - assert (out / "_COMMUNITY_Backend.md").exists() - assert (out / "MyOwnNote.md").read_text().strip() == "mine" - - -def test_to_obsidian_removed_node_returning_is_writable_again(capsys): - """#1896 follow-on: a node that disappears and later returns must be writable - again. Before the fix, the manifest was rewritten to only this run's files, so - the orphaned note was disowned and the returning node's write was skipped as a - 'pre-existing user file' forever.""" - import networkx as nx - GA, commA = _two_node_graph() - GB = nx.Graph() - GB.add_node("n1", label="Database", community=0, source_file="app/db.py", type="code") - commB = {0: ["n1"]} - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "obsidian" - to_obsidian(GA, commA, str(out), community_labels={0: "Backend"}) - to_obsidian(GB, commB, str(out), community_labels={0: "Backend"}) - assert not (out / "Server.md").exists() # pruned while absent - capsys.readouterr() - to_obsidian(GA, commA, str(out), community_labels={0: "Backend"}) - # returned node's note exists with current content, written this run - assert (out / "Server.md").exists() - assert "# Server" in (out / "Server.md").read_text() - captured = capsys.readouterr() - assert "skipped" not in captured.err.lower() - - -# ── Case-only-distinct labels must not collide on case-insensitive filesystems ── - -def _case_collision_graph(): - """Two nodes whose labels differ only by case - on macOS/APFS and Windows/NTFS - their notes resolve to the same path unless the dedup map folds case.""" - return build_from_json({ - "nodes": [ - {"id": "n1", "label": "References", "file_type": "code", "source_file": "a.py"}, - {"id": "n2", "label": "references", "file_type": "document", "source_file": "b.md"}, +def _graph(): + return graph_from_payload( + [ + {"id": "a", "label": "Auth", "file_type": "code", "source_file": "auth.py"}, + {"id": "b", "label": "API", "file_type": "code", "source_file": "api.py"}, ], - "edges": [], - }) - - -def test_to_obsidian_case_only_distinct_labels_dont_overwrite(): - """Both notes must survive as separate files. On a case-insensitive filesystem - a missing suffix silently overwrites the first note (fewer files than nodes); - on a case-sensitive one it writes two stems equal under .lower(). Assert both: - every node note is on disk, and no two stems collide case-insensitively.""" - G = _case_collision_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp) - notes = [p for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")] - assert len(notes) == G.number_of_nodes(), [p.name for p in notes] - lowered = [p.stem.lower() for p in notes] - assert len(set(lowered)) == len(lowered), [p.name for p in notes] - # the suffixed name must be the expected one, not merely distinct - assert sorted(p.stem for p in notes) == ["References", "references_1"], [p.name for p in notes] - - -def test_to_obsidian_generated_suffix_doesnt_overwrite_literal(): - """A generated `_1` suffix must not collide with a node whose literal label is - already that suffixed name. With labels [dup, dup, dup_1] the second `dup` - becomes `dup_1`, which would clobber the third node unless the candidate is - re-checked. This collides on case-sensitive filesystems too, so it guards the - dedup loop independently of case-folding.""" - G = build_from_json({ - "nodes": [ - {"id": "a", "label": "dup", "file_type": "code", "source_file": "a.py"}, - {"id": "b", "label": "dup", "file_type": "code", "source_file": "b.py"}, - {"id": "c", "label": "dup_1", "file_type": "code", "source_file": "c.py"}, - ], - "edges": [], - }) - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp) - notes = [p for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")] - assert len(notes) == 3, [p.name for p in notes] - assert len({p.stem.lower() for p in notes}) == 3, [p.name for p in notes] - - -def test_to_canvas_case_only_distinct_labels_get_distinct_files(): - """Canvas file-node references for case-only-distinct labels must be distinct - case-insensitively, else both cards point at one overwritten note.""" - G = _case_collision_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "graph.canvas" - to_canvas(G, communities, str(out)) - data = json.loads(out.read_text()) - files = [n["file"] for n in data["nodes"] if n.get("type") == "file"] - lowered = [f.lower() for f in files] - assert len(set(lowered)) == len(lowered), files - - -def test_obsidian_canvas_filenames_agree(): - """The CLI calls to_obsidian and to_canvas separately with no shared map, so - they must independently produce the same node->filename mapping - otherwise a - canvas card points at a note file that doesn't exist on disk.""" - G = _case_collision_graph() - communities = cluster(G) - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp) - note_stems = {p.stem for p in Path(tmp).rglob("*.md") if not p.name.startswith("_COMMUNITY")} - out = Path(tmp) / "graph.canvas" - to_canvas(G, communities, str(out)) - data = json.loads(out.read_text()) - canvas_stems = {Path(n["file"]).stem for n in data["nodes"] if n.get("type") == "file"} - assert canvas_stems <= note_stems, (sorted(canvas_stems), sorted(note_stems)) - - -def test_to_obsidian_community_notes_case_collision(): - """Two community labels differing only by case must each get their own - `_COMMUNITY_*.md` overview note. This path had no dedup at all, so even - same-case duplicate labels previously overwrote silently.""" - G = build_from_json({ - "nodes": [ - {"id": "n1", "label": "alpha", "file_type": "code", "source_file": "a.py"}, - {"id": "n2", "label": "beta", "file_type": "code", "source_file": "b.py"}, - ], - "edges": [], - }) - communities = {0: ["n1"], 1: ["n2"]} - labels = {0: "API", 1: "Api"} - with tempfile.TemporaryDirectory() as tmp: - to_obsidian(G, communities, tmp, community_labels=labels) - comm = [p for p in Path(tmp).rglob("_COMMUNITY_*.md")] - assert len(comm) == 2, [p.name for p in comm] - lowered = [p.stem.lower() for p in comm] - assert len(set(lowered)) == len(lowered), [p.name for p in comm] - - -# ── Issue #834: backup_if_protected ────────────────────────────────────────── - -def test_backup_no_graph_json(tmp_path): - """No graph.json → no backup.""" - from graphify.export import backup_if_protected - assert backup_if_protected(tmp_path) is None - - -def test_backup_no_markers(tmp_path): - """graph.json present but no sentinel and no curated labels → no backup.""" - from graphify.export import backup_if_protected - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - assert backup_if_protected(tmp_path) is None - - -def test_backup_semantic_marker(tmp_path): - """graph.json + .graphify_semantic_marker → backup taken.""" - from graphify.export import backup_if_protected - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / "GRAPH_REPORT.md").write_text("# Report") - (tmp_path / ".graphify_semantic_marker").write_text('{"output_tokens": 1234}') - result = backup_if_protected(tmp_path) - assert result is not None - assert result.is_dir() - assert (result / "graph.json").exists() - assert (result / "GRAPH_REPORT.md").exists() - assert (result / ".graphify_semantic_marker").exists() - - -def test_backup_curated_labels(tmp_path): - """graph.json + non-default label in .graphify_labels.json → backup taken.""" - import json - from graphify.export import backup_if_protected - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_labels.json").write_text(json.dumps({"0": "Auth Pipeline", "1": "Community 1"})) - result = backup_if_protected(tmp_path) - assert result is not None - - -def test_backup_default_labels_only(tmp_path): - """All-default labels → no backup (not curated).""" - import json - from graphify.export import backup_if_protected - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_labels.json").write_text(json.dumps({"0": "Community 0", "1": "Community 1"})) - assert backup_if_protected(tmp_path) is None - - -def test_backup_same_day_no_accumulation(tmp_path): - """Same content on same day returns existing backup dir without re-copying.""" - from graphify.export import backup_if_protected - from datetime import date - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_semantic_marker").write_text("{}") - b1 = backup_if_protected(tmp_path) - b2 = backup_if_protected(tmp_path) - assert b1 is not None and b2 is not None - assert b1 == b2 # same dir, no _2 accumulation - assert b1.name == date.today().isoformat() - - -def test_backup_same_day_changed_content(tmp_path): - """Changed graph.json on same day overwrites the existing backup in place.""" - from graphify.export import backup_if_protected - from datetime import date - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_semantic_marker").write_text("{}") - b1 = backup_if_protected(tmp_path) - (tmp_path / "graph.json").write_text('{"nodes":[{"id":"x"}],"links":[]}') - b2 = backup_if_protected(tmp_path) - assert b1 == b2 # still one folder per day - assert (b2 / "graph.json").read_text() == '{"nodes":[{"id":"x"}],"links":[]}' - - -def test_backup_env_disable(tmp_path, monkeypatch): - """GRAPHIFY_NO_BACKUP=1 disables backup entirely.""" - from graphify.export import backup_if_protected - monkeypatch.setenv("GRAPHIFY_NO_BACKUP", "1") - (tmp_path / "graph.json").write_text('{"nodes":[],"links":[]}') - (tmp_path / ".graphify_semantic_marker").write_text("{}") - assert backup_if_protected(tmp_path) is None - - -def _mkG(n): - import networkx as nx - G = nx.Graph() - for i in range(n): - G.add_node(f"n{i}", label=f"n{i}", community=0) - return G - - -def test_to_json_refuses_shrink(tmp_path): - """#479: refuse to silently overwrite an existing graph with fewer nodes.""" - p = tmp_path / "graph.json" - json.dump({"nodes": [{"id": f"n{i}"} for i in range(5)]}, p.open("w")) - assert to_json(_mkG(2), {}, str(p), force=False) is False - assert to_json(_mkG(2), {}, str(p), force=True) is True # force overrides - - -def test_to_json_fails_safe_on_corrupt_existing(tmp_path): - """A non-empty but unparseable existing graph.json (corrupt or mid-write) - must NOT be silently overwritten — we can't verify the new graph isn't a - partial shrink, so fail safe (refuse) unless force is given.""" - p = tmp_path / "graph.json" - p.write_text("{ this has content but is not valid json") - assert to_json(_mkG(10), {}, str(p), force=False) is False - assert to_json(_mkG(10), {}, str(p), force=True) is True - - -def test_to_json_proceeds_on_empty_existing(tmp_path): - """An empty/whitespace existing file has no nodes to lose, so it is not a - shrink risk — the write proceeds.""" - p = tmp_path / "graph.json" - p.write_text("") - assert to_json(_mkG(3), {}, str(p), force=False) is True - data = json.loads(p.read_text()) - assert len(data["nodes"]) == 3 - - -def test_to_html_handles_null_source_file_and_label(tmp_path): - """#1775: a node with source_file=None or label=None must not crash to_html - (synthetic/aggregate nodes legitimately carry null source_file; JSON `null` - survives .get()'s default). Regression guard — fixed via sanitize_label's - None-coercion + the str(source_file or "") call-site guard.""" - import networkx as nx - G = nx.Graph() - G.add_node("n1", label="Foo", source_file=None, community=0) - G.add_node("n2", label=None, source_file="a.py", community=0) - G.add_node("n3", label=None, source_file=None, community=0) - out = tmp_path / "graph.html" - to_html(G, {0: ["n1", "n2", "n3"]}, str(out)) - assert out.exists() and out.stat().st_size > 0 - - -def test_existing_graph_node_count(tmp_path): - from graphify.export import existing_graph_node_count, MALFORMED_GRAPH - p = tmp_path / "graph.json" - assert existing_graph_node_count(p) is None # absent -> nothing to protect - p.write_text("", encoding="utf-8") - assert existing_graph_node_count(p) is None # empty -> nothing to protect - # Non-empty but unparseable must fail CLOSED (sentinel), matching to_json's - # #479 guard — a corrupt/mid-write file could be hiding a complete graph. - p.write_text("{not json", encoding="utf-8") - assert existing_graph_node_count(p) is MALFORMED_GRAPH # malformed -> fail closed - p.write_text('{"nodes": "notalist"}', encoding="utf-8") - assert existing_graph_node_count(p) is MALFORMED_GRAPH # structurally wrong -> fail closed - p.write_text('{"nodes": [{"id": "a"}, {"id": "b"}], "links": []}', encoding="utf-8") - assert existing_graph_node_count(p) == 2 # valid + [{"source": "a", "target": "b", "relation": "calls", "confidence": "INFERRED", "weight": 0.5}], + kind="digraph", + ) + + +def test_native_presentation_exports(tmp_path): + graph = _graph() + communities = {0: ["a"], 1: ["b"]} + labels = {0: "Security", 1: "Interface"} + html = tmp_path / "graph.html" + graphml = tmp_path / "graph.graphml" + cypher = tmp_path / "graph.cypher" + canvas = tmp_path / "graph.canvas" + vault = tmp_path / "vault" + to_html(graph, communities, str(html), community_labels=labels) + to_graphml(graph, communities, str(graphml)) + to_cypher(graph, str(cypher)) + to_canvas(graph, communities, str(canvas), community_labels=labels) + assert to_obsidian(graph, communities, str(vault), community_labels=labels) == 4 + assert "Auth" in html.read_text() and "calls" in html.read_text() + ET.parse(graphml) + assert "INFERRED" in graphml.read_text() + assert "MERGE" in cypher.read_text() and "CALLS" in cypher.read_text() + canvas_nodes = json.loads(canvas.read_text())["nodes"] + assert len([node for node in canvas_nodes if node["type"] == "file"]) == 2 + assert (vault / "Auth.md").is_file() and (vault / "API.md").is_file() + + +def test_typed_ids_export_without_collision(tmp_path): + graph = graph_from_payload( + [{"id": 1, "label": "Integer"}, {"id": "1", "label": "String"}], + [{"source": 1, "target": "1", "relation": "links"}], + ) + output = tmp_path / "typed.graphml" + to_graphml(graph, {0: [1, "1"]}, str(output)) + root = ET.parse(output).getroot() + ids = [element.attrib["id"] for element in root.findall(".//{*}node")] + assert len(ids) == len(set(ids)) == 2 + + +def test_export_escapes_hostile_labels(tmp_path): + graph = graph_from_payload( + [{"id": "a", "label": "", "source_file": "a.py"}] + ) + output = tmp_path / "safe.html" + to_html(graph, {0: ["a"]}, str(output)) + text = output.read_text() + assert "") -def test_validate_url_rejects_empty_scheme(): +@pytest.mark.parametrize("url", ["file:///etc/passwd", "ftp://example.com/a", "data:text/plain,x"]) +def test_url_rejects_non_http_schemes(url): with pytest.raises(ValueError): - validate_url("//no-scheme.example.com") - - -# --------------------------------------------------------------------------- -# safe_fetch - scheme and redirect guards (mocked network) -# --------------------------------------------------------------------------- - -def _make_mock_response(content: bytes, status: int = 200): - mock = MagicMock() - mock.__enter__ = lambda s: s - mock.__exit__ = MagicMock(return_value=False) - mock.status = status - mock.code = status - chunks = [content[i:i+65536] for i in range(0, len(content), 65536)] + [b""] - mock.read.side_effect = chunks - return mock - - -def test_safe_fetch_rejects_file_url(): - with pytest.raises(ValueError, match="file"): - safe_fetch("file:///etc/passwd") - -def test_safe_fetch_rejects_ftp_url(): - with pytest.raises(ValueError, match="ftp"): - safe_fetch("ftp://example.com/file.zip") + validate_url(url) -def test_safe_fetch_returns_bytes(tmp_path): - mock_resp = _make_mock_response(b"hello world") - with patch("graphify.security._build_opener") as mock_opener_fn: - mock_opener = MagicMock() - mock_opener.open.return_value = mock_resp - mock_opener_fn.return_value = mock_opener - result = safe_fetch("https://example.com/") - assert result == b"hello world" -def test_safe_fetch_raises_on_non_2xx(): - mock_resp = _make_mock_response(b"Not Found", status=404) - with patch("graphify.security._build_opener") as mock_opener_fn: - mock_opener = MagicMock() - mock_opener.open.return_value = mock_resp - mock_opener_fn.return_value = mock_opener - with pytest.raises(urllib.error.HTTPError): - safe_fetch("https://example.com/missing") - -def test_safe_fetch_raises_on_size_exceeded(): - # Build a response larger than max_bytes - big_chunk = b"x" * 65_537 - mock_resp = MagicMock() - mock_resp.__enter__ = lambda s: s - mock_resp.__exit__ = MagicMock(return_value=False) - mock_resp.status = 200 - mock_resp.code = 200 - # Return the chunk twice so total > max_bytes=65536 - mock_resp.read.side_effect = [big_chunk, big_chunk, b""] - - with patch("graphify.security._build_opener") as mock_opener_fn: - mock_opener = MagicMock() - mock_opener.open.return_value = mock_resp - mock_opener_fn.return_value = mock_opener +def test_safe_fetch_caps_streamed_response(): + response = MagicMock() + response.__enter__ = lambda value: value + response.__exit__ = MagicMock(return_value=False) + response.status = 200 + response.read.side_effect = [b"x" * 9, b""] + with patch("graphify.security.validate_url"), patch("graphify.security._build_opener") as factory: + factory.return_value.open.return_value = response with pytest.raises(OSError, match="size limit"): - safe_fetch("https://example.com/huge", max_bytes=65_536) - - -# --------------------------------------------------------------------------- -# safe_fetch_text -# --------------------------------------------------------------------------- - -def test_safe_fetch_text_decodes_utf8(): - content = "héllo wörld".encode("utf-8") - mock_resp = _make_mock_response(content) - with patch("graphify.security._build_opener") as mock_opener_fn: - mock_opener = MagicMock() - mock_opener.open.return_value = mock_resp - mock_opener_fn.return_value = mock_opener - result = safe_fetch_text("https://example.com/") - assert result == "héllo wörld" - -def test_safe_fetch_text_replaces_bad_bytes(): - bad = b"hello \xff world" - mock_resp = _make_mock_response(bad) - with patch("graphify.security._build_opener") as mock_opener_fn: - mock_opener = MagicMock() - mock_opener.open.return_value = mock_resp - mock_opener_fn.return_value = mock_opener - result = safe_fetch_text("https://example.com/") - assert "hello" in result - assert "world" in result - assert "\xff" not in result + safe_fetch("https://example.com", max_bytes=8) -# --------------------------------------------------------------------------- -# validate_graph_path -# --------------------------------------------------------------------------- - -def test_validate_graph_path_allows_inside_base(tmp_path): - base = tmp_path / "graphify-out" - base.mkdir() - graph = base / "graph.json" - graph.write_text("{}") - result = validate_graph_path(str(graph), base=base) - assert result == graph.resolve() - -def test_validate_graph_path_blocks_traversal(tmp_path): - base = tmp_path / "graphify-out" - base.mkdir() - evil = tmp_path / "graphify-out" / ".." / "etc_passwd" - with pytest.raises(ValueError, match="escapes"): - validate_graph_path(str(evil), base=base) - -def test_validate_graph_path_requires_base_exists(tmp_path): - base = tmp_path / "graphify-out" # not created - with pytest.raises(ValueError, match="does not exist"): - validate_graph_path(str(base / "graph.json"), base=base) - -def test_validate_graph_path_raises_if_file_missing(tmp_path): - base = tmp_path / "graphify-out" - base.mkdir() +def test_native_store_validation_and_legacy_rejection(tmp_path): + store = tmp_path / "graph.helix" + store.mkdir() + assert validate_store_path(store) == store.resolve() + legacy = tmp_path / "legacy.json" + legacy.write_text("{}") + with pytest.raises(ValueError, match="obsolete"): + validate_store_path(legacy) with pytest.raises(FileNotFoundError): - validate_graph_path(str(base / "missing.json"), base=base) - -def test_validate_graph_path_default_base_discovers_output_dir(tmp_path): - """With base omitted, the output dir is discovered by walking the path's - parents for the configured output-dir name (default 'graphify-out').""" - base = tmp_path / "graphify-out" - base.mkdir() - graph = base / "graph.json" - graph.write_text("{}") - assert validate_graph_path(str(graph)) == graph.resolve() - -def test_validate_graph_path_default_base_honours_graphify_out_override(tmp_path, monkeypatch): - """The base=None discovery must honour GRAPHIFY_OUT, not the hardcoded - 'graphify-out' literal — otherwise a renamed output dir validates against the - wrong base or raises spuriously (#1423).""" - monkeypatch.setattr("graphify.security.GRAPHIFY_OUT_NAME", "custom-out") - monkeypatch.setattr("graphify.security.GRAPHIFY_OUT", "custom-out") - out = tmp_path / "custom-out" - out.mkdir() - graph = out / "graph.json" - graph.write_text("{}") - # No base passed → must discover custom-out by name rather than graphify-out. - assert validate_graph_path(str(graph)) == graph.resolve() + validate_store_path(tmp_path / "missing.helix") -# --------------------------------------------------------------------------- -# sanitize_label -# --------------------------------------------------------------------------- - -def test_sanitize_label_passthrough_html_chars(): - # sanitize_label does NOT HTML-escape — callers that inject into HTML must - # wrap with html.escape() themselves (e.g. the title in to_html()) - assert sanitize_label("") - assert "<" in result - assert ">" in result - assert ""}) - assert isinstance(out, dict) - assert "<" in out["k"] - - -def test_sanitize_metadata_value_recurses_into_list(): - out = _sanitize_metadata_value(["", "", ""]) - assert isinstance(out, list) - assert all("<" in s for s in out) - - -def test_sanitize_metadata_value_caps_list_length(): - huge = list(range(_METADATA_MAX_LIST_ITEMS * 3)) - out = _sanitize_metadata_value(huge) - assert isinstance(out, list) - assert len(out) == _METADATA_MAX_LIST_ITEMS - - -def test_sanitize_metadata_value_converts_tuple_to_list(): - out = _sanitize_metadata_value(("a", "b")) - assert isinstance(out, list) - assert out == ["a", "b"] - - -def test_sanitize_metadata_none_returns_empty_dict(): - assert sanitize_metadata(None) == {} - - -def test_sanitize_metadata_drops_empty_key(): - # Empty key (after control-char strip) is dropped. - out = sanitize_metadata({"\x00": "v", "k": "v2"}) - assert "\x00" not in out - assert out.get("k") == "v2" - assert len(out) == 1 - - -def test_sanitize_metadata_sanitizes_keys(): - out = sanitize_metadata({"": "v"}) - assert "" not in out - assert any("<" in k for k in out.keys()) - - -def test_sanitize_metadata_recursive_nested(): - raw: dict[str, Any] = { - "outer": { - "inner": "", - "list": ["a", "", 99, None, True], - }, - "scalar": 42, - } - out = sanitize_metadata(raw) - assert isinstance(out["outer"], dict) - inner = out["outer"] - assert isinstance(inner, dict) - assert "<" in inner["inner"] - items = inner["list"] - assert isinstance(items, list) - assert items[0] == "a" - assert "<" in items[1] - assert items[2] == 99 - assert items[3] is None - assert items[4] is True - assert out["scalar"] == 42 - - -def test_sanitize_metadata_bool_not_coerced_to_int(): - # bool is an int subclass — order of isinstance checks must preserve bool. - out = sanitize_metadata({"flag_t": True, "flag_f": False, "num": 1}) - assert out["flag_t"] is True - assert out["flag_f"] is False - assert out["num"] == 1 + assert "\x00" not in sanitize_label("a\x00b") + assert len(sanitize_label("x" * 1000)) <= 256 + cleaned = sanitize_metadata({"nested": {"value": "x" * 10000}}) + assert len(cleaned["nested"]["value"]) < 10000 diff --git a/tests/test_semantic_cache_out_root.py b/tests/test_semantic_cache_out_root.py index 2bde8d3c9..b004acf4e 100644 --- a/tests/test_semantic_cache_out_root.py +++ b/tests/test_semantic_cache_out_root.py @@ -1,221 +1,83 @@ -"""Regression tests for #1990 and #1991. - -#1990 — `graphify extract --out` saves recovery checkpoints in the wrong directory. - save_semantic_cache must write to cache_root (the --out dir), not root - (the corpus dir), when they differ. - -#1991 — Final semantic-cache save resolves source_file paths against out_root - and silently writes 0 entries when corpus files do not exist there. - Fixed by passing root=target (corpus), cache_root=out_root to - save_semantic_cache so source_file resolution and cache placement are - independently anchored. -""" -import warnings -from pathlib import Path - -import pytest - -from graphify.cache import ( - check_semantic_cache, - file_hash, - load_cached, - save_semantic_cache, -) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _semantic_dir(root: Path, mode: str | None = None) -> Path: - kind = "semantic" if mode is None else f"semantic-{mode}" - return root / "graphify-out" / "cache" / kind - - -def _count_cache_files(base: Path) -> int: - """Count .json files under a cache dir (recursively, excluding .tmp).""" - if not base.is_dir(): - return 0 - return sum(1 for _ in base.rglob("*.json")) - - -# --------------------------------------------------------------------------- -# #1990 — checkpoint writes to cache_root, not corpus root -# --------------------------------------------------------------------------- - - -def test_save_semantic_cache_writes_to_cache_root_not_corpus(tmp_path): - """When cache_root differs from root, cache files must land under cache_root.""" - corpus = tmp_path / "corpus" - out = tmp_path / "out" - corpus.mkdir() - out.mkdir() - - doc = corpus / "report.md" - doc.write_text("# Report\nSome content here.") - - nodes = [{"id": "n1", "label": "Report", "source_file": str(doc)}] - edges: list = [] - - saved = save_semantic_cache( - nodes, edges, root=corpus, cache_root=out - ) - - assert saved == 1, "expected 1 entry written" - # Cache file must be under the out dir, not the corpus dir - assert _count_cache_files(_semantic_dir(out)) == 1 - assert _count_cache_files(_semantic_dir(corpus)) == 0 - - -def test_save_semantic_cache_no_corpus_graphify_out_created(tmp_path): - """With cache_root set, no graphify-out/ dir should be created inside corpus.""" - corpus = tmp_path / "corpus" - out = tmp_path / "out" - corpus.mkdir() - out.mkdir() - - doc = corpus / "notes.md" - doc.write_text("Notes content.") - - save_semantic_cache( - [{"id": "x", "label": "X", "source_file": str(doc)}], - [], - root=corpus, - cache_root=out, - ) - - assert not (corpus / "graphify-out").exists(), ( - "graphify-out/ must not be created inside the corpus when cache_root is set" - ) +"""Semantic extraction cache lives in Helix generation state, never sidecars.""" +from pathlib import Path -# --------------------------------------------------------------------------- -# #1990 — checkpoint written with cache_root is found on recovery read -# --------------------------------------------------------------------------- +from graphify.cache import check_semantic_cache, save_semantic_cache -def test_checkpoint_with_cache_root_is_found_by_check_semantic_cache(tmp_path): - """A checkpoint saved with cache_root can be retrieved by check_semantic_cache - using the same root/cache_root split — simulating recovery after interruption.""" +def test_semantic_cache_round_trips_through_generation_mapping(tmp_path: Path): corpus = tmp_path / "corpus" - out = tmp_path / "out" corpus.mkdir() - out.mkdir() - doc = corpus / "paper.md" doc.write_text("Some academic content.") + cache: dict = {} - nodes = [{"id": "p1", "label": "Paper", "source_file": str(doc)}] - - # Simulate a checkpoint written mid-run (merge_existing path) - save_semantic_cache( - nodes, [], + saved = save_semantic_cache( + [{"id": "p1", "label": "Paper", "source_file": str(doc)}], + [], root=corpus, - cache_root=out, merge_existing=True, allowed_source_files=[doc], + cache=cache, ) - - # Recovery read: check_semantic_cache with the same root/cache_root split - # cli.py now uses (root=target, cache_root=out_root). - cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( - [str(doc)], root=corpus, cache_root=out + nodes, edges, hyperedges, uncached = check_semantic_cache( + [str(doc)], cache, root=corpus ) - assert len(uncached) == 0, f"expected cache hit, got miss for: {uncached}" - assert any(n.get("id") == "p1" for n in cached_nodes) - - -# --------------------------------------------------------------------------- -# #1991 — final save resolves source_file against corpus root, not out_root -# --------------------------------------------------------------------------- + assert saved == 1 + assert not uncached and not edges and not hyperedges + assert nodes[0]["id"] == "p1" + assert Path(nodes[0]["source_file"]) == doc -def test_final_save_with_out_root_populates_cache(tmp_path): - """When root=corpus and cache_root=out, source_file resolution must use - corpus as anchor so p.is_file() succeeds and the entry is written.""" +def test_semantic_cache_uses_corpus_root_for_relative_sources(tmp_path: Path): corpus = tmp_path / "corpus" - out = tmp_path / "out" corpus.mkdir() - out.mkdir() - doc = corpus / "report.md" doc.write_text("# Annual Report\nKey findings.") - - # This is the pattern that was broken: source_file is relative to corpus, - # but was previously resolved against out_root, making p.is_file() → False. - nodes = [{"id": "r1", "label": "AnnualReport", "source_file": "report.md"}] + cache: dict = {} saved = save_semantic_cache( - nodes, [], + [{"id": "r1", "source_file": "report.md"}], + [], root=corpus, - cache_root=out, allowed_source_files=[doc], + cache=cache, ) - assert saved == 1, ( - "final save must write 1 entry when root=corpus and source_file is " - "relative to corpus — resolving against out_root caused 0 writes (#1991)" - ) - assert _count_cache_files(_semantic_dir(out)) == 1 - - -def test_final_save_with_wrong_root_emits_warning(tmp_path): - """Passing root=out_root (the old broken behaviour) silently writes 0 entries; - the fix adds a RuntimeWarning when ALL groups are dropped.""" - corpus = tmp_path / "corpus" - out = tmp_path / "out" - corpus.mkdir() - out.mkdir() - - doc = corpus / "report.md" - doc.write_text("# Report") - - # Old (broken) call: root=out, no cache_root — the corpus-relative - # source_file is resolved against out where the file doesn't exist - # → 0 entries written. - nodes = [{"id": "r1", "label": "R", "source_file": "report.md"}] - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - saved = save_semantic_cache(nodes, [], root=out) - - assert saved == 0 - # The new warning must fire when every group is dropped - assert any( - "#1991" in str(warning.message) for warning in w - ), "expected RuntimeWarning mentioning #1991 when all groups are silently skipped" - - -# --------------------------------------------------------------------------- -# Backward compat — omitting cache_root keeps legacy behaviour -# --------------------------------------------------------------------------- + assert saved == 1 + assert len(cache) == 1 + assert not (corpus / "graphify-out").exists() -def test_save_semantic_cache_backward_compat_no_cache_root(tmp_path): - """When cache_root is omitted, cache files still land under root (unchanged).""" +def test_semantic_cache_is_inert_without_generation_mapping(tmp_path: Path): root = tmp_path / "project" root.mkdir() - doc = root / "main.md" doc.write_text("Main content.") - nodes = [{"id": "m1", "label": "Main", "source_file": str(doc)}] - - saved = save_semantic_cache(nodes, [], root=root) + saved = save_semantic_cache( + [{"id": "m1", "source_file": str(doc)}], [], root=root + ) - assert saved == 1 - assert _count_cache_files(_semantic_dir(root)) == 1 + assert saved == 0 + assert list(root.iterdir()) == [doc] -def test_extract_corpus_parallel_accepts_cache_root_kwarg(): - """extract_corpus_parallel must accept a cache_root kwarg without raising - (import + signature check — no actual LLM call).""" - import inspect - from graphify.llm import extract_corpus_parallel +def test_semantic_cache_invalidates_content_changes(tmp_path: Path): + root = tmp_path / "project" + root.mkdir() + doc = root / "main.md" + doc.write_text("Original body.") + cache: dict = {} + save_semantic_cache( + [{"id": "m1", "source_file": str(doc)}], [], root=root, cache=cache + ) + doc.write_text("Changed body.") - sig = inspect.signature(extract_corpus_parallel) - assert "cache_root" in sig.parameters, ( - "extract_corpus_parallel must expose cache_root so cli.py can plumb " - "out_root through to _checkpoint_chunk (#1990)" + nodes, _edges, _hyperedges, uncached = check_semantic_cache( + [str(doc)], cache, root=root ) + + assert not nodes + assert uncached == [str(doc)] diff --git a/tests/test_semantic_cleanup.py b/tests/test_semantic_cleanup.py index de13bd245..a4080b68a 100644 --- a/tests/test_semantic_cleanup.py +++ b/tests/test_semantic_cleanup.py @@ -52,7 +52,7 @@ def test_validate_semantic_fragment_rejects_path_separator_in_id(): def test_validate_semantic_fragment_accepts_unknown_file_type(): - """An unknown/synonym file_type is NOT a validation failure: build_from_json + """An unknown/synonym file_type is NOT a validation failure: build_from_extraction coerces any value via _FILE_TYPE_SYNONYMS (unknown -> "concept", #840), so rejecting a whole chunk over it would be pure data loss. file_type carries no security risk, so it is left to build's coercion rather than gated here.""" diff --git a/tests/test_semantic_id_remap_root.py b/tests/test_semantic_id_remap_root.py index 3a5fee626..08a60a232 100644 --- a/tests/test_semantic_id_remap_root.py +++ b/tests/test_semantic_id_remap_root.py @@ -11,7 +11,7 @@ from pathlib import Path -from graphify.build import _semantic_id_remap, build_from_json +from graphify.build import _semantic_id_remap, build_from_extraction from graphify.extractors.base import _file_stem @@ -28,7 +28,7 @@ def test_semantic_id_remap_root_equal_source_file_no_crash(): assert "some_concept" not in remap -def test_build_from_json_with_root_level_concept_node(): +def test_build_from_extraction_with_root_level_concept_node(): root = "/proj" combined = { "nodes": [ @@ -39,8 +39,8 @@ def test_build_from_json_with_root_level_concept_node(): ], "edges": [], } - G = build_from_json(combined, root=root) # previously crashed here - assert G.number_of_nodes() == 2 + G = build_from_extraction(combined, root=root) # previously crashed here + assert G.node_count == 2 def test_normal_semantic_remap_still_works(): diff --git a/tests/test_semantic_similarity.py b/tests/test_semantic_similarity.py index 55d9cce34..6bfd0345b 100644 --- a/tests/test_semantic_similarity.py +++ b/tests/test_semantic_similarity.py @@ -1,194 +1,20 @@ -"""Tests for semantically_similar_to edge support.""" -import networkx as nx -import pytest -from graphify.build import build_from_json -from graphify.analyze import surprising_connections, _surprise_score -from graphify.report import generate +from graphify.analyze import surprising_connections +from tests.native_helpers import make_loaded -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _make_extraction_with_semantic_edge(): - """Two nodes in separate files connected by a semantically_similar_to edge.""" - return { - "nodes": [ - {"id": "a_validate_input", "label": "validate_input", "file_type": "code", - "source_file": "auth/validators.py", "source_location": "L5"}, - {"id": "b_check_input", "label": "check_input", "file_type": "code", - "source_file": "api/checks.py", "source_location": "L12"}, - ], - "edges": [ - { - "source": "a_validate_input", - "target": "b_check_input", - "relation": "semantically_similar_to", - "confidence": "INFERRED", - "confidence_score": 0.82, - "source_file": "auth/validators.py", - "source_location": None, - "weight": 0.82, - } +def test_semantic_relation_is_native_label_and_analysis_property(tmp_path): + graph = make_loaded( + tmp_path, + nodes=[ + {"id": "a", "label": "A", "source_file": "src/a.py"}, + {"id": "b", "label": "B", "source_file": "docs/b.md"}, ], - "input_tokens": 100, - "output_tokens": 50, - } - - -def _make_graph_with_semantic_edge(): - return build_from_json(_make_extraction_with_semantic_edge()) - - -def _make_two_edge_graph(): - """Graph with one semantically_similar_to edge and one references edge, both cross-file.""" - G = nx.Graph() - for nid, label, src in [ - ("a", "ValidateInput", "auth/validators.py"), - ("b", "CheckInput", "api/checks.py"), - ("c", "LoadConfig", "config/loader.py"), - ("d", "ReadConfig", "utils/reader.py"), - ]: - G.add_node(nid, label=label, source_file=src, file_type="code") - # semantically_similar_to edge - G.add_edge("a", "b", relation="semantically_similar_to", confidence="INFERRED", - confidence_score=0.82, source_file="auth/validators.py", weight=0.82, - _src="a", _tgt="b") - # plain references edge (same confidence tier) - G.add_edge("c", "d", relation="references", confidence="INFERRED", - confidence_score=0.7, source_file="config/loader.py", weight=0.7, - _src="c", _tgt="d") - return G - - -# --------------------------------------------------------------------------- -# Test 1: semantically_similar_to passes through build_from_json without being dropped -# --------------------------------------------------------------------------- - -def test_semantic_edge_survives_build_from_json(): - G = _make_graph_with_semantic_edge() - assert G.number_of_edges() == 1 - u, v, data = next(iter(G.edges(data=True))) - assert data["relation"] == "semantically_similar_to" - - -def test_semantic_edge_nodes_present(): - G = _make_graph_with_semantic_edge() - assert "a_validate_input" in G.nodes - assert "b_check_input" in G.nodes - - -# --------------------------------------------------------------------------- -# Test 2: confidence_score is preserved for semantically_similar_to edges -# --------------------------------------------------------------------------- - -def test_semantic_edge_confidence_score_preserved(): - G = _make_graph_with_semantic_edge() - u, v, data = next(iter(G.edges(data=True))) - assert data.get("confidence_score") == pytest.approx(0.82) - assert data.get("confidence") == "INFERRED" - - -# --------------------------------------------------------------------------- -# Test 3: surprising_connections scores semantically_similar_to edges higher -# than references edges with the same community membership -# --------------------------------------------------------------------------- - -def test_semantic_edge_scores_higher_than_references(): - G = _make_two_edge_graph() - communities = {0: ["a", "b"], 1: ["c", "d"]} - node_community = {"a": 0, "b": 0, "c": 1, "d": 1} - - score_sem, reasons_sem = _surprise_score( - G, "a", "b", G.edges["a", "b"], node_community, - "auth/validators.py", "api/checks.py" - ) - score_ref, _ = _surprise_score( - G, "c", "d", G.edges["c", "d"], node_community, - "config/loader.py", "utils/reader.py" - ) - assert score_sem > score_ref - - -def test_semantic_edge_reason_mentions_similarity(): - G = _make_two_edge_graph() - communities = {0: ["a", "b"], 1: ["c", "d"]} - node_community = {"a": 0, "b": 0, "c": 1, "d": 1} - - _, reasons = _surprise_score( - G, "a", "b", G.edges["a", "b"], node_community, - "auth/validators.py", "api/checks.py" - ) - assert any("similar" in r for r in reasons) - - -# --------------------------------------------------------------------------- -# Test 4: report renders [semantically similar] tag for these edges -# --------------------------------------------------------------------------- - -def _make_report_with_semantic_surprise(): - G = _make_graph_with_semantic_edge() - communities = {0: ["a_validate_input", "b_check_input"]} - cohesion = {0: 0.5} - labels = {0: "Validators"} - gods = [] - surprises = [ - { - "source": "validate_input", - "target": "check_input", + edges=[{ + "source": "a", "target": "b", "relation": "semantically_similar_to", "confidence": "INFERRED", - "confidence_score": 0.82, - "source_files": ["auth/validators.py", "api/checks.py"], - "why": "semantically similar concepts with no structural link", - } - ] - detection = {"total_files": 2, "total_words": 500, "needs_graph": True, "warning": None} - tokens = {"input": 100, "output": 50} - return generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") - - -def test_report_renders_semantically_similar_tag(): - report = _make_report_with_semantic_surprise() - assert "[semantically similar]" in report - - -def test_report_semantic_tag_on_correct_line(): - report = _make_report_with_semantic_surprise() - for line in report.splitlines(): - if "semantically_similar_to" in line: - assert "[semantically similar]" in line - break - else: - pytest.fail("No line with semantically_similar_to found in report") - - -def test_report_no_semantic_tag_for_other_relations(): - """Non-semantic edges must not get the [semantically similar] tag.""" - G = nx.Graph() - for nid, label, src in [ - ("x", "Alpha", "repo1/a.py"), - ("y", "Beta", "repo2/b.py"), - ]: - G.add_node(nid, label=label, source_file=src, file_type="code") - G.add_edge("x", "y", relation="references", confidence="EXTRACTED", - confidence_score=1.0, source_file="repo1/a.py", weight=1.0) - - communities = {0: ["x", "y"]} - cohesion = {0: 0.5} - labels = {0: "Misc"} - gods = [] - surprises = [ - { - "source": "Alpha", - "target": "Beta", - "relation": "references", - "confidence": "EXTRACTED", - "source_files": ["repo1/a.py", "repo2/b.py"], - "why": "cross-file connection", - } - ] - detection = {"total_files": 2, "total_words": 200, "needs_graph": True, "warning": None} - tokens = {"input": 50, "output": 25} - report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") - assert "[semantically similar]" not in report + }], + ).graph + assert graph.edges()[0].label == "semantically_similar_to" + surprises = surprising_connections(graph, {0: ["a"], 1: ["b"]}) + assert surprises[0]["relation"] == "semantically_similar_to" diff --git a/tests/test_serve.py b/tests/test_serve.py index 302787187..1db700cf5 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -1,71 +1,145 @@ """Tests for serve.py - MCP graph query helpers (no mcp package required).""" import json +from typing import Any + import pytest -import networkx as nx -from networkx.readwrite import json_graph +from tests.native_helpers import triangle, make_loaded +from graphify.helix.model import node_attributes from graphify.serve import ( _communities_from_graph, - _score_nodes, - _score_query, - _compute_idf, + _score_nodes as _native_score_nodes, + _score_query as _native_score_query, + _compute_idf as _native_compute_idf, _EXACT_MATCH_BONUS, _SOURCE_MATCH_BONUS, - _pick_seeds, - _bfs, - _dfs, - _find_node, - _trigrams, - _node_search_text, - _get_trigram_index, - _trigram_candidates, - _filter_graph_by_context, + _pick_seeds as _native_pick_seeds, + _bfs as _native_bfs, + _dfs as _native_dfs, + _find_node as _native_find_node, + _filter_graph_by_context as _native_filter_graph_by_context, _infer_context_filters, _query_terms, - _query_graph_text, + _query_graph_text as _native_query_graph_text, _resolve_context_filters, - _subgraph_to_text, + _subgraph_to_text as _native_subgraph_to_text, _load_graph, _community_header, _search_tokens, ) -def _make_graph() -> nx.Graph: - G = nx.Graph() - G.add_node("n1", label="extract", source_file="extract.py", source_location="L10", community=0) - G.add_node("n2", label="cluster", source_file="cluster.py", source_location="L5", community=0) - G.add_node("n3", label="build", source_file="build.py", source_location="L1", community=1) - G.add_node("n4", label="report", source_file="report.py", source_location="L1", community=1) - G.add_node("n5", label="isolated", source_file="other.py", source_location="L1", community=2) - G.add_edge("n1", "n2", relation="calls", confidence="INFERRED", context="call") - G.add_edge("n2", "n3", relation="imports", confidence="EXTRACTED", context="import") - G.add_edge("n3", "n4", relation="uses", confidence="EXTRACTED") - return G +def _graph(loaded): + return getattr(loaded, "graph", loaded) -# --- _communities_from_graph --- +def _query(loaded): + query = getattr(loaded, "query", None) + assert query is not None, "query helpers must be tested through a real LoadedGraph" + return query -def test_communities_from_graph_basic(): - G = _make_graph() - communities = _communities_from_graph(G) - assert 0 in communities - assert 1 in communities - assert "n1" in communities[0] - assert "n2" in communities[0] - assert "n3" in communities[1] - -def test_communities_from_graph_no_community_attr(): - G = nx.Graph() - G.add_node("a", label="foo") # no community attr - communities = _communities_from_graph(G) - assert communities == {} - -def test_communities_from_graph_isolated(): - G = _make_graph() - communities = _communities_from_graph(G) - assert 2 in communities - assert "n5" in communities[2] + +def _score_nodes(loaded, terms): + return _native_score_nodes(_graph(loaded), terms, native_query=_query(loaded)) + + +def _score_query(loaded, terms, *, collect_per_term_seeds): + return _native_score_query( + _graph(loaded), + terms, + collect_per_term_seeds=collect_per_term_seeds, + native_query=_query(loaded), + ) + + +def _compute_idf(loaded, terms): + return _native_compute_idf(_graph(loaded), terms, _query(loaded)) + + +def _find_node(loaded, label): + return _native_find_node(_graph(loaded), label, native_query=_query(loaded)) + + +def _bfs(loaded, start_nodes, depth, context_filters=None): + return _native_bfs( + _graph(loaded), + start_nodes, + depth, + context_filters, + native_query=_query(loaded), + ) + + +def _dfs(loaded, start_nodes, depth, context_filters=None): + return _native_dfs( + _graph(loaded), + start_nodes, + depth, + context_filters, + native_query=_query(loaded), + ) + + +def _query_graph_text(loaded, question, **kwargs): + return _native_query_graph_text( + _graph(loaded), question, native_query=_query(loaded), **kwargs + ) + + +def _pick_seeds(scored, *args, G=None, **kwargs): + return _native_pick_seeds( + scored, *args, G=_graph(G) if G is not None else None, **kwargs + ) + + +def _filter_graph_by_context(loaded, context_filters): + _native_graph, filters = _native_filter_graph_by_context( + _graph(loaded), context_filters + ) + return loaded, filters + + +def _subgraph_to_text(loaded, nodes, edges, *args, **kwargs): + return _native_subgraph_to_text( + _graph(loaded), nodes, edges, *args, **kwargs + ) + + +def _edge_records(loaded, *pairs): + G = _graph(loaded) + records = [] + for source, target in pairs: + edge_ids = G.edges_between(source, target) or G.edges_between(target, source) + records.append(G.edge(edge_ids[0])) + return records + + +def _make_graph() -> Any: + return make_loaded( + nodes=[ + {"id": "n1", "label": "extract", "source_file": "extract.py", "source_location": "L10", "community": 0}, + {"id": "n2", "label": "cluster", "source_file": "cluster.py", "source_location": "L5", "community": 0}, + {"id": "n3", "label": "build", "source_file": "build.py", "source_location": "L1", "community": 1}, + {"id": "n4", "label": "report", "source_file": "report.py", "source_location": "L1", "community": 1}, + {"id": "n5", "label": "isolated", "source_file": "other.py", "source_location": "L1", "community": 2}, + ], + edges=[ + {"source": "n1", "target": "n2", "relation": "calls", "confidence": "INFERRED", "context": "call"}, + {"source": "n2", "target": "n3", "relation": "imports", "confidence": "EXTRACTED", "context": "import"}, + {"source": "n3", "target": "n4", "relation": "uses", "confidence": "EXTRACTED"}, + ], + ) + + +def test_native_dfs_respects_depth(tmp_path): + graph = triangle(tmp_path) + nodes, _ = _dfs(graph, ["a"], 0) + assert nodes == {"a"} + + +def test_loaded_snapshot_has_no_python_runtime_cache(): + loaded = make_loaded(nodes=[{"id": "snapshot", "label": "Snapshot"}]) + assert not hasattr(loaded.graph, "_graphify_cache") # --- _score_nodes --- @@ -105,19 +179,26 @@ def test_score_nodes_multiword_exact_label_outranks_superset(): arbitrary node-id sort, and a wrong/disconnected endpoint was chosen. The full-query tier in _score_nodes must make the exact label win strictly. """ - G = nx.Graph() # Reproduce the real graph: norm_label keeps punctuation (strip_diacritics + # lower, NOT tokenized), so the ':' survives. A tokenized query can never # equal that, which is exactly why the first-cut fix was a no-op for # punctuated labels. The exact node must still win via the label's tokenized # form. - def _add(nid, label, src): - G.add_node(nid, label=label, norm_label=label.lower(), - source_file=src, community=0) - - _add("exact", "UOCE: Dehumidifier Driver", "uoce_dehumidifier.yaml") - _add("super", "UOCE: Dehumidifier Driver State Machine", "uoce_dehumidifier.yaml") - _add("decoy", "Dehumidifier Driver Helper", "uoce_dehumidifier.yaml") + nodes = [ + { + "id": nid, + "label": label, + "norm_label": label.lower(), + "source_file": "uoce_dehumidifier.yaml", + "community": 0, + } + for nid, label in ( + ("exact", "UOCE: Dehumidifier Driver"), + ("super", "UOCE: Dehumidifier Driver State Machine"), + ("decoy", "Dehumidifier Driver Helper"), + ) + ] + G = make_loaded(nodes=nodes) # CLI resolves endpoints as [t.lower() for t in label.split()]. scored = _score_nodes(G, [t.lower() for t in "UOCE: Dehumidifier Driver".split()]) @@ -140,25 +221,23 @@ def test_score_nodes_coverage_lone_generic_exact_hit_loses_to_multi_term_match() (the realistic case) to pin that source-path hits do not count as coverage and hand the collision its exact tier back. """ - G = nx.Graph() - - def _add(nid, label, src): - G.add_node(nid, label=label, norm_label=label.lower(), - source_file=src, community=0) - - _add("target", "ClientLive.Index", "lib/clients_live/index.ex") - _add("form", "ClientLive.Form", "lib/clients_live/form.ex") - _add("show", "ClientLive.Show", "lib/clients_live/show.ex") + nodes = [ + {"id": "target", "label": "ClientLive.Index", "norm_label": "clientlive.index", "source_file": "lib/clients_live/index.ex", "community": 0}, + {"id": "form", "label": "ClientLive.Form", "norm_label": "clientlive.form", "source_file": "lib/clients_live/form.ex", "community": 0}, + {"id": "show", "label": "ClientLive.Show", "norm_label": "clientlive.show", "source_file": "lib/clients_live/show.ex", "community": 0}, + ] # Same-named tiny leaf functions: "list" == bare label fires the exact # tier. Placed in the target's own directory so their source paths also # substring-match the query term "clients": a path hit must not inflate # the coverage that multiplies the exact tier. for i in range(3): - _add(f"leaf{i}", "list()", f"lib/clients_live/helpers{i}.ex") + nodes.append({"id": f"leaf{i}", "label": "list()", "norm_label": "list()", "source_file": f"lib/clients_live/helpers{i}.ex", "community": 0}) # Filler making "list" a common (low-IDF) token, as in a real graph where # list()/get()/new() style names are ubiquitous. for i in range(24): - _add(f"filler{i}", f"shopping list {i}", f"lib/filler{i}.ex") + label = f"shopping list {i}" + nodes.append({"id": f"filler{i}", "label": label, "norm_label": label, "source_file": f"lib/filler{i}.ex", "community": 0}) + G = make_loaded(nodes=nodes) # The user pastes the real identifier plus context words; tokenization # yields 5 terms: clientlive, index, clients, list, columns. @@ -193,8 +272,7 @@ def test_find_node_ignores_trailing_punctuation(): def test_find_node_matches_full_punctuated_unicode_label(): - G = nx.Graph() - G.add_node("n1", label="Skill /auditar — Auditoría inquisitiva de enlaces") + G = make_loaded(nodes=[{"id": "n1", "label": "Skill /auditar — Auditoría inquisitiva de enlaces"}]) assert _find_node(G, "Skill /auditar — Auditoría inquisitiva de enlaces") == ["n1"] @@ -202,11 +280,10 @@ def test_find_node_matches_full_punctuated_unicode_label(): def test_find_node_matches_punctuated_file_label_exactly(): # #1704: an exactly-typed punctuated file label must resolve through explain, # just like it does through path/query. - G = nx.Graph() - G.add_node("f1", label="blockStream.ts", norm_label="blockstream.ts", - source_file="lib/blockStream.ts", source_location="L1") - G.add_node("f2", label="blockStream.test.ts", norm_label="blockstream.test.ts", - source_file="lib/blockStream.test.ts", source_location="L1") + G = make_loaded(nodes=[ + {"id": "f1", "label": "blockStream.ts", "norm_label": "blockstream.ts", "source_file": "lib/blockStream.ts", "source_location": "L1"}, + {"id": "f2", "label": "blockStream.test.ts", "norm_label": "blockstream.test.ts", "source_file": "lib/blockStream.test.ts", "source_location": "L1"}, + ]) assert _find_node(G, "blockStream.ts")[0] == "f1" assert _find_node(G, "blockStream.test.ts")[0] == "f2" @@ -217,95 +294,39 @@ def test_find_node_resolves_when_label_and_norm_label_diverge(): # `norm_label` diverge, only the symmetric `norm_query == norm_label` match # resolves it. Here label tokenizes to "blockstream" but norm_label is # "blockstream.ts" — this fails without the norm_query path. - G = nx.Graph() - G.add_node("n1", label="BlockStream", norm_label="blockstream.ts", - source_file="lib/x.ts", source_location="L1") + G = make_loaded(nodes=[{"id": "n1", "label": "BlockStream", "norm_label": "blockstream.ts", "source_file": "lib/x.ts", "source_location": "L1"}]) assert _find_node(G, "blockStream.ts") == ["n1"] -# --- trigram candidate prefilter (the trigram index that shrinks the O(N) scan) --- - +# --- native predicate candidate selection --- -def _force_full_scan(monkeypatch): - """Disable the prefilter so a call exercises the original full-node scan.""" - monkeypatch.setattr("graphify.serve._trigram_candidates", lambda *a, **k: None) - -def _make_big_graph(n: int = 150) -> nx.Graph: +def _make_big_graph(n: int = 150) -> Any: """A graph large enough that the selectivity guard lets the fast-path fire for rare terms and fall back for common ones. Most labels share the 'item'/'node' stem (common), plus a few distinctive rare labels and one punctuated label.""" - G = nx.Graph() - for i in range(n): - G.add_node(f"id{i}", label=f"item node {i}", source_file=f"pkg/item_{i}.py") - G.add_node("rareA", label="ZebraQuokkaWidget", source_file="zoo/zqw.py") - G.add_node("rareB", label="MarmosetGadget handler", source_file="zoo/marmoset.py") - G.add_node("punct", label="Foo.Bar:Baz", source_file="pkg/foobar.py") - return G - - -def test_trigrams_basic(): - assert _trigrams("foobar") == {"foo", "oob", "oba", "bar"} - assert _trigrams("ab") == {"ab"} # <3 chars -> whole string is the key - assert _trigrams("") == set() - - -def test_node_search_text_includes_all_matched_fields(): - G = _make_big_graph() - text = _node_search_text(G.nodes["punct"], "punct") - # norm_label, tokenized label, nid, raw source, and tokenized source are all - # present, NUL-separated so trigrams can't span fields. - parts = text.split("\x00") - assert parts[0] == "foo.bar:baz" # norm_label (punctuation kept) - assert parts[1] == "foo bar baz" # label_tokens (tokenized) - assert parts[2] == "punct" # nid - assert parts[3] == "pkg/foobar.py" # source_file - assert parts[4] == "pkg foobar py" # source_file tokens - - -def test_trigram_candidates_fast_path_fires_for_rare_term(): - G = _make_big_graph() - cand = _trigram_candidates(G, ["zebraquokkawidget"]) - assert cand is not None # selective -> fast-path used - assert "rareA" in cand - assert len(cand) < G.number_of_nodes() # a real shrink, not the whole graph - - -def test_trigram_candidates_falls_back_on_common_term(): - G = _make_big_graph() - # 'item' is in the label of every one of the 150 'item node N' nodes -> the - # rarest trigram is still common -> guard returns None (full-scan fallback). - assert _trigram_candidates(G, ["item"]) is None - - -def test_trigram_candidates_falls_back_on_short_token(): - G = _make_big_graph() - assert _trigram_candidates(G, ["ab"]) is None # <3 chars -> can't trigram-filter + nodes = [ + {"id": f"id{i}", "label": f"item node {i}", "source_file": f"pkg/item_{i}.py"} + for i in range(n) + ] + nodes.extend([ + {"id": "rareA", "label": "ZebraQuokkaWidget", "source_file": "zoo/zqw.py"}, + {"id": "rareB", "label": "MarmosetGadget handler", "source_file": "zoo/marmoset.py"}, + {"id": "punct", "label": "Foo.Bar:Baz", "source_file": "pkg/foobar.py"}, + ]) + return make_loaded(nodes=nodes) -def test_score_nodes_prefilter_is_identical_to_full_scan(monkeypatch): +def test_native_candidates_find_rare_label_without_python_index(): G = _make_big_graph() - queries = ["zebraquokkawidget", "marmosetgadget handler", "foo bar baz", - "item", "node 42", "nonexistentxyz"] - for q in queries: - terms = _query_terms(q) - fast = _score_nodes(G, terms) - _force_full_scan(monkeypatch) - full = _score_nodes(G, terms) - monkeypatch.undo() - assert fast == full, f"prefilter diverged from full scan for {q!r}" + candidates = _query(G).candidate_ids(["zebraquokkawidget"]) + assert candidates == ["rareA"] -def test_find_node_prefilter_is_identical_to_full_scan(monkeypatch): +def test_native_candidates_cover_common_and_short_terms(): G = _make_big_graph() - # includes the punctuated label, exercised via its tokenized (label_tokens) form - for label in ["ZebraQuokkaWidget", "MarmosetGadget handler", "Foo Bar Baz", - "item node 7", "missing"]: - fast = _find_node(G, label) - _force_full_scan(monkeypatch) - full = _find_node(G, label) - monkeypatch.undo() - assert fast == full, f"_find_node prefilter diverged (order!) for {label!r}" + assert len(_query(G).candidate_ids(["item"])) == 150 + assert _query(G).candidate_ids(["ab"]) == [] def test_find_node_label_tokens_branch_covered_by_index(): @@ -317,38 +338,18 @@ def test_find_node_label_tokens_branch_covered_by_index(): def test_find_node_source_file_path_prefers_file_level_node(): - G = _make_big_graph() source_file = "app/api/example/route.ts" # Insert the function node first to prove source-file lookup reorders the # file-level node ahead of other nodes from the same file. - G.add_node( - "example_route_get", - label="GET()", - source_file=source_file, - source_location="L42", - ) - G.add_node( - "example_route", - label="route.ts", - source_file=source_file, - source_location="L1", - ) + G = make_loaded(nodes=[ + {"id": "example_route_get", "label": "GET()", "source_file": source_file, "source_location": "L42"}, + {"id": "example_route", "label": "route.ts", "source_file": source_file, "source_location": "L1"}, + ]) matches = _find_node(G, source_file) assert matches[0] == "example_route" assert "example_route_get" in matches - - -def test_trigram_index_cached_and_rebuilt_per_graph(): - G = _make_big_graph() - idx1 = _get_trigram_index(G) - assert idx1 is _get_trigram_index(G) # cached on the same graph object - assert G.graph["_trigram_index"] is idx1 - G2 = _make_big_graph() - assert _get_trigram_index(G2) is not idx1 # a fresh graph rebuilds (reload safety) - - def test_query_terms_strips_search_punctuation(): # "what" is a question stopword (dropped); punctuation is still stripped from "extract?". assert _query_terms("what calls extract?") == ["calls", "extract"] @@ -384,12 +385,16 @@ def test_pick_seeds_german_query_seeds_content_node_not_heading_noise(): """End-to-end for #1900: a German question over a graph with German heading-noise nodes must seed on the content noun, not on nodes that happen to contain 'die'/'wie'/'wird'.""" - G = nx.DiGraph() - G.add_node("cfg", label="Die Konfiguration", source_file="docs/konfiguration.md") - G.add_node("sec", label="Wie wird gesichert", source_file="docs/sicherheit.md") - G.add_node("auth", label="Authentifizierung", source_file="src/auth.py") - G.add_node("helper", label="login_helper", source_file="src/auth.py") - G.add_edge("helper", "auth") + G = make_loaded( + kind="digraph", + nodes=[ + {"id": "cfg", "label": "Die Konfiguration", "source_file": "docs/konfiguration.md"}, + {"id": "sec", "label": "Wie wird gesichert", "source_file": "docs/sicherheit.md"}, + {"id": "auth", "label": "Authentifizierung", "source_file": "src/auth.py"}, + {"id": "helper", "label": "login_helper", "source_file": "src/auth.py"}, + ], + edges=[{"source": "helper", "target": "auth"}], + ) q = "Wie funktioniert die Authentifizierung?" terms = _query_terms(q) @@ -423,8 +428,7 @@ def cut(self, text): def test_query_graph_text_keeps_short_non_english_terms(): - G = nx.Graph() - G.add_node("frontend", label="前端", source_file="docs/前端.md", source_location="L1", community=0) + G = make_loaded(nodes=[{"id": "frontend", "label": "前端", "source_file": "docs/前端.md", "source_location": "L1", "community": 0}]) text = _query_graph_text(G, "前端", mode="bfs", depth=1) assert "No matching nodes found." not in text assert "NODE 前端" in text @@ -463,16 +467,16 @@ def test_bfs_returns_edges(): G = _make_graph() visited, edges = _bfs(G, ["n1"], depth=1) assert len(edges) >= 1 - assert any(u == "n1" or v == "n1" for u, v in edges) + assert any(edge.source == "n1" or edge.target == "n1" for edge in edges) def test_filter_graph_by_context_limits_traversal(): G = _make_graph() - filtered = _filter_graph_by_context(G, ["call"]) - visited, edges = _bfs(filtered, ["n1"], depth=2) + filtered, filters = _filter_graph_by_context(G, ["call"]) + visited, edges = _bfs(filtered, ["n1"], depth=2, context_filters=filters) assert "n2" in visited assert "n3" not in visited - assert edges == [("n1", "n2")] + assert [(edge.source, edge.target) for edge in edges] == [("n1", "n2")] # --- _dfs --- @@ -494,26 +498,26 @@ def test_dfs_full_chain(): def test_subgraph_to_text_contains_labels(): G = _make_graph() - text = _subgraph_to_text(G, {"n1", "n2"}, [("n1", "n2")]) + text = _subgraph_to_text(G, {"n1", "n2"}, _edge_records(G, ("n1", "n2"))) assert "extract" in text assert "cluster" in text def test_subgraph_to_text_truncates(): G = _make_graph() # Very small budget forces truncation - text = _subgraph_to_text(G, {"n1", "n2", "n3", "n4"}, [("n1", "n2")], token_budget=1) + text = _subgraph_to_text(G, {"n1", "n2", "n3", "n4"}, _edge_records(G, ("n1", "n2")), token_budget=1) assert "truncated" in text def test_subgraph_to_text_edge_included(): G = _make_graph() - text = _subgraph_to_text(G, {"n1", "n2"}, [("n1", "n2")]) + text = _subgraph_to_text(G, {"n1", "n2"}, _edge_records(G, ("n1", "n2"))) assert "EDGE" in text assert "calls" in text def test_subgraph_to_text_includes_edge_context(): G = _make_graph() - text = _subgraph_to_text(G, {"n1", "n2"}, [("n1", "n2")]) + text = _subgraph_to_text(G, {"n1", "n2"}, _edge_records(G, ("n1", "n2"))) assert "context=call" in text @@ -523,10 +527,15 @@ def test_subgraph_to_text_annotates_node_with_learning_status(): """An annotated node gets a `learning=` suffix inside its NODE bracket; an un-annotated node gets none.""" G = _make_graph() - G.graph["_learning_overlay"] = { + overlay = { "n1": {"status": "preferred", "stale": False}, } - text = _subgraph_to_text(G, {"n1", "n2"}, [("n1", "n2")]) + text = _subgraph_to_text( + G, + {"n1", "n2"}, + _edge_records(G, ("n1", "n2")), + learning_overlay=overlay, + ) lines = {l.split()[1]: l for l in text.splitlines() if l.startswith("NODE ")} assert "learning=preferred]" in lines["extract"] assert "learning=" not in lines["cluster"] # un-annotated node @@ -534,8 +543,8 @@ def test_subgraph_to_text_annotates_node_with_learning_status(): def test_subgraph_to_text_marks_stale_status(): G = _make_graph() - G.graph["_learning_overlay"] = {"n1": {"status": "contested", "stale": True}} - text = _subgraph_to_text(G, {"n1"}, []) + overlay = {"n1": {"status": "contested", "stale": True}} + text = _subgraph_to_text(G, {"n1"}, [], learning_overlay=overlay) assert "learning=contested:stale]" in text @@ -550,10 +559,16 @@ def test_subgraph_to_text_learning_suffix_counts_against_budget(): assert "truncated" not in _subgraph_to_text(G, {"n1", "n2", "n3"}, [], token_budget=budget) # ...but once every node carries a learning= suffix, the same budget overflows. - G.graph["_learning_overlay"] = { + overlay = { n: {"status": "preferred", "stale": False} for n in ("n1", "n2", "n3") } - annotated = _subgraph_to_text(G, {"n1", "n2", "n3"}, [], token_budget=budget) + annotated = _subgraph_to_text( + G, + {"n1", "n2", "n3"}, + [], + token_budget=budget, + learning_overlay=overlay, + ) assert "learning=preferred" in annotated assert "truncated" in annotated @@ -561,7 +576,7 @@ def test_subgraph_to_text_learning_suffix_counts_against_budget(): def test_subgraph_to_text_no_overlay_is_unchanged(): """With no overlay on the graph, NODE lines carry no learning= suffix.""" G = _make_graph() - text = _subgraph_to_text(G, {"n1", "n2"}, [("n1", "n2")]) + text = _subgraph_to_text(G, {"n1", "n2"}, _edge_records(G, ("n1", "n2"))) assert "learning=" not in text @@ -584,114 +599,76 @@ def test_query_graph_text_heuristic_context_filter_changes_traversal(): # --- _load_graph --- def test_load_graph_roundtrip(tmp_path): - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") - p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) - G2 = _load_graph(str(p)) - assert G2.number_of_nodes() == G.number_of_nodes() - assert G2.number_of_edges() == G.number_of_edges() + loaded = make_loaded( + tmp_path, + nodes=[{"id": "a", "label": "A"}, {"id": "b", "label": "B"}], + edges=[{"source": "a", "target": "b", "relation": "calls"}], + ) + loaded_again = _load_graph(str(loaded.store_path)) + assert loaded_again.graph.node_count == 2 + assert loaded_again.graph.edge_count == 1 def test_load_graph_missing_file(tmp_path): graphify_dir = tmp_path / "graphify-out" graphify_dir.mkdir() with pytest.raises(SystemExit): - _load_graph(str(graphify_dir / "nonexistent.json")) - - -def test_load_graph_rejects_oversized_file(monkeypatch, tmp_path, capsys): - # #F4: oversized graph.json must fail fast (SystemExit) with a clear error. - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") - p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 16) - with pytest.raises(SystemExit): - _load_graph(str(p)) - err = capsys.readouterr().err - assert "exceeds" in err - assert "byte cap" in err - - -def test_load_graph_accepts_under_cap(monkeypatch, tmp_path): - # Verifies the cap path does not regress the normal load. - G = _make_graph() - data = json_graph.node_link_data(G, edges="links") - p = tmp_path / "graph.json" - p.write_text(json.dumps(data)) - # Cap well above the actual file size — load proceeds. - monkeypatch.setattr("graphify.security._MAX_GRAPH_FILE_BYTES", 10 * 1024 * 1024) - G2 = _load_graph(str(p)) - assert G2.number_of_nodes() == G.number_of_nodes() + _load_graph(str(graphify_dir / "nonexistent.helix")) # --- #874: MCP hot-reload --- def _write_graph(path, nodes: list[str]) -> None: - """Write a minimal graph.json with the given node IDs.""" - G = nx.DiGraph() - for n in nodes: - G.add_node(n, label=n, community=0) - data = json_graph.node_link_data(G, edges="links") - path.write_text(json.dumps(data), encoding="utf-8") + """Activate a minimal native generation with the given node IDs.""" + make_loaded(path.parent, nodes=[{"id": n, "label": n, "community": 0} for n in nodes], kind="digraph") def test_maybe_reload_detects_graph_change(tmp_path): - """serve() picks up a new graph.json written after startup (#874).""" - import time - from unittest.mock import patch + """serve() sees a newly activated native generation after startup (#874).""" out = tmp_path / "graphify-out" out.mkdir() - graph_path = out / "graph.json" + graph_path = out / "graph.helix" _write_graph(graph_path, ["alpha", "beta"]) # Bootstrap _load_graph + _communities_from_graph to verify the reload path - G1 = _load_graph(str(graph_path)) - assert set(G1.nodes()) == {"alpha", "beta"} + first = _load_graph(str(graph_path)) + assert {node.id for node in first.graph.nodes()} == {"alpha", "beta"} - # Simulate file changing (bump mtime by touching) - time.sleep(0.01) _write_graph(graph_path, ["alpha", "beta", "gamma"]) - G2 = _load_graph(str(graph_path)) - assert "gamma" in G2.nodes() - + second = _load_graph(str(graph_path)) + assert second.generation != first.generation + assert second.graph.contains_node("gamma") -def test_load_graph_cache_key_changes_with_content(tmp_path): - """mtime_ns + size uniquely identifies a graph version (#874).""" - import time +def test_load_graph_generation_changes_with_content(tmp_path): + """Atomic activation gives every graph version a distinct generation ID.""" out = tmp_path / "graphify-out" out.mkdir() - graph_path = out / "graph.json" + graph_path = out / "graph.helix" _write_graph(graph_path, ["a"]) - - s1 = graph_path.stat() - key1 = (s1.st_mtime_ns, s1.st_size) - - time.sleep(0.01) + first = _load_graph(str(graph_path)) _write_graph(graph_path, ["a", "b"]) - - s2 = graph_path.stat() - key2 = (s2.st_mtime_ns, s2.st_size) - - assert key1 != key2, "stat key must change when file content changes" + second = _load_graph(str(graph_path)) + assert first.generation != second.generation # --- IDF weighting tests (#897) --- -def _make_noisy_graph() -> nx.Graph: +def _make_noisy_graph() -> Any: """20 error-handler nodes + 1 rare identifier: FooBarService.""" - G = nx.Graph() + nodes = [] + edges = [] for i in range(20): - G.add_node(f"err{i}", label=f"error_handler_{i}", source_file=f"err{i}.py", community=0) + nodes.append({"id": f"err{i}", "label": f"error_handler_{i}", "source_file": f"err{i}.py", "community": 0}) if i > 0: - G.add_edge(f"err{i-1}", f"err{i}", relation="calls", confidence="EXTRACTED") - G.add_node("fbs", label="FooBarService", source_file="service.py", community=1) - G.add_node("fbs_dep", label="ServiceClient", source_file="client.py", community=1) - G.add_edge("fbs", "fbs_dep", relation="uses", confidence="EXTRACTED") - return G + edges.append({"source": f"err{i-1}", "target": f"err{i}", "relation": "calls", "confidence": "EXTRACTED"}) + nodes.extend([ + {"id": "fbs", "label": "FooBarService", "source_file": "service.py", "community": 1}, + {"id": "fbs_dep", "label": "ServiceClient", "source_file": "client.py", "community": 1}, + ]) + edges.append({"source": "fbs", "target": "fbs_dep", "relation": "uses", "confidence": "EXTRACTED"}) + return make_loaded(nodes=nodes, edges=edges) def test_idf_downweights_common_terms(): @@ -705,20 +682,10 @@ def test_idf_downweights_common_terms(): ) -def test_idf_cached_on_graph(): - """IDF results are stored in G.graph so repeated queries don't recompute.""" +def test_idf_uses_native_counts_without_python_cache(): G = _make_graph() - _score_nodes(G, ["extract"]) - assert "_idf_cache" in G.graph - assert "extract" in G.graph["_idf_cache"] - - -def test_idf_new_graph_starts_fresh(): - """Two separate graph instances must not share an IDF cache.""" - G1 = _make_graph() - G2 = _make_graph() - _score_nodes(G1, ["extract"]) - assert "_idf_cache" not in G2.graph + assert _query(G).document_frequencies(["extract"]) == {"extract": 1} + assert not hasattr(_graph(G), "_graphify_cache") def test_idf_rare_term_gets_high_weight(): @@ -733,10 +700,10 @@ def test_idf_rare_term_gets_high_weight(): def test_idf_common_term_gets_low_weight(): """A term matching most nodes should get IDF < 1.""" import math - G = nx.Graph() - # 'handle' in every node label - for i in range(20): - G.add_node(f"n{i}", label=f"handle_{i}", source_file=f"f{i}.py") + G = make_loaded(nodes=[ + {"id": f"n{i}", "label": f"handle_{i}", "source_file": f"f{i}.py"} + for i in range(20) + ]) idf = _compute_idf(G, ["handle"]) assert idf["handle"] < 1.0 @@ -787,14 +754,18 @@ def test_pick_seeds_diversity_recovers_starved_term(monkeypatch): G/best_seed_by_term, the 20%-gap cutoff discards the relevant candidate entirely; with them, it is recovered as a guaranteed per-term seed. """ - G = nx.DiGraph() # "unrelated" is an exact label match for the query term "unrelated" and # has no connection to the actually-relevant "target" node. - G.add_node("noise", label="unrelated", source_file="design_tokens.json") # "target" only substring-matches the query term "widget" via its label. - G.add_node("target", label="rate_limit_widget", source_file="src/widget.py") - G.add_node("other", label="something_else", source_file="src/other.py") - G.add_edge("other", "target") + G = make_loaded( + kind="digraph", + nodes=[ + {"id": "noise", "label": "unrelated", "source_file": "design_tokens.json"}, + {"id": "target", "label": "rate_limit_widget", "source_file": "src/widget.py"}, + {"id": "other", "label": "something_else", "source_file": "src/other.py"}, + ], + edges=[{"source": "other", "target": "target"}], + ) terms = ["unrelated", "widget"] # `_score_query` does the combined scoring and the per-term singleton @@ -818,10 +789,10 @@ def test_pick_seeds_dedups_homonymous_generic_labels(): """Many nodes sharing one generic label (e.g. framework `GET` handlers) must contribute at most ONE seed, not consume every slot (#1766). A distinct, relevant label still gets its own seed.""" - G = nx.DiGraph() - for i in range(5): - G.add_node(f"get{i}", label="GET", source_file=f"routes/r{i}.py") - G.add_node("um", label="users_model", source_file="models/users.py") + G = make_loaded(kind="digraph", nodes=[ + *[{"id": f"get{i}", "label": "GET", "source_file": f"routes/r{i}.py"} for i in range(5)], + {"id": "um", "label": "users_model", "source_file": "models/users.py"}, + ]) # Score all the GET nodes above users_model so, pre-fix, they'd take every slot. scored = [(1000.0, f"get{i}") for i in range(5)] + [(900.0, "um")] seeds = _pick_seeds(scored, G=G) @@ -833,10 +804,11 @@ def test_pick_seeds_dedups_homonymous_generic_labels(): def test_pick_seeds_dedup_key_is_case_and_diacritic_normalized(): """`GET`/`Get`/`get` are the same generic label and must dedup together.""" - G = nx.DiGraph() - G.add_node("a", label="GET", source_file="a.py") - G.add_node("b", label="Get", source_file="b.py") - G.add_node("c", label="get", source_file="c.py") + G = make_loaded(kind="digraph", nodes=[ + {"id": "a", "label": "GET", "source_file": "a.py"}, + {"id": "b", "label": "Get", "source_file": "b.py"}, + {"id": "c", "label": "get", "source_file": "c.py"}, + ]) scored = [(1000.0, "a"), (990.0, "b"), (980.0, "c")] seeds = _pick_seeds(scored, G=G) assert len(seeds) == 1, f"case-variant duplicates not collapsed: {seeds}" @@ -845,11 +817,14 @@ def test_pick_seeds_dedup_key_is_case_and_diacritic_normalized(): def test_pick_seeds_per_term_guarantee_does_not_reintroduce_generic_dupe(monkeypatch): """The per-term guarantee loop must honor the same per-label cap, so it can't add a second `GET` after dedup already seeded one (#1766).""" - G = nx.DiGraph() - for i in range(3): - G.add_node(f"get{i}", label="GET", source_file=f"r{i}.py") - G.add_node("um", label="users_model", source_file="users.py") - G.add_edge("um", "get0") + G = make_loaded( + kind="digraph", + nodes=[ + *[{"id": f"get{i}", "label": "GET", "source_file": f"r{i}.py"} for i in range(3)], + {"id": "um", "label": "users_model", "source_file": "users.py"}, + ], + edges=[{"source": "um", "target": "get0"}], + ) terms = ["get", "users"] qs = _score_query(G, terms, collect_per_term_seeds=True) seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) @@ -862,10 +837,11 @@ def test_score_nodes_scores_identical_labels_equally(): (shared by shortest_path / explain endpoint resolution): two nodes with the SAME label must receive the SAME score for a query, i.e. the fix lives in seed selection, not in the shared scorer (#1766 followup).""" - G = nx.DiGraph() - G.add_node("g1", label="GET", source_file="a.py") - G.add_node("g2", label="GET", source_file="b.py") - G.add_node("g3", label="GET", source_file="c.py") + G = make_loaded(kind="digraph", nodes=[ + {"id": "g1", "label": "GET", "source_file": "a.py"}, + {"id": "g2", "label": "GET", "source_file": "b.py"}, + {"id": "g3", "label": "GET", "source_file": "c.py"}, + ]) by_id = {nid: s for s, nid in _score_nodes(G, ["get"])} assert by_id["g1"] == by_id["g2"] == by_id["g3"], ( f"identical-label nodes scored differently: {by_id}" @@ -877,7 +853,7 @@ def test_score_nodes_scores_identical_labels_equally(): def test_subgraph_to_text_truncation_hint_is_actionable(): """Truncation message must tell Claude what to do, not just say truncated.""" G = _make_graph() - text = _subgraph_to_text(G, {"n1", "n2", "n3", "n4"}, [("n1", "n2")], token_budget=1) + text = _subgraph_to_text(G, {"n1", "n2", "n3", "n4"}, _edge_records(G, ("n1", "n2")), token_budget=1) assert "truncated" in text assert "get_node" in text or "context_filter" in text @@ -894,15 +870,17 @@ def test_query_seeds_from_identifier_not_noise(): def test_query_graph_text_parameter_type_context_filter_changes_traversal(): - import networkx as nx - from graphify.serve import _query_graph_text - - graph = nx.Graph() - graph.add_node("process", label="process", source_file="sample.cs", source_location="L20") - graph.add_node("payload", label="Payload", source_file="sample.cs", source_location="L5") - graph.add_node("other", label="PayloadFactory", source_file="sample.cs", source_location="L40") - graph.add_edge("process", "payload", relation="references", context="parameter_type", confidence="EXTRACTED") - graph.add_edge("process", "other", relation="calls", context="call", confidence="EXTRACTED") + graph = make_loaded( + nodes=[ + {"id": "process", "label": "process", "source_file": "sample.cs", "source_location": "L20"}, + {"id": "payload", "label": "Payload", "source_file": "sample.cs", "source_location": "L5"}, + {"id": "other", "label": "PayloadFactory", "source_file": "sample.cs", "source_location": "L40"}, + ], + edges=[ + {"source": "process", "target": "payload", "relation": "references", "context": "parameter_type", "confidence": "EXTRACTED"}, + {"source": "process", "target": "other", "relation": "calls", "context": "call", "confidence": "EXTRACTED"}, + ], + ) text = _query_graph_text(graph, "who accepts Payload", context_filters=["parameter_type"]) @@ -912,7 +890,6 @@ def test_query_graph_text_parameter_type_context_filter_changes_traversal(): def test_query_graph_text_context_filter_aliases_resolve(): - import networkx as nx from graphify.serve import _normalize_context_filters assert _normalize_context_filters(["param"]) == ["parameter_type"] @@ -976,9 +953,10 @@ def test_query_terms_chinese_no_jieba_fallback(monkeypatch): def test_score_nodes_chinese_substring_match(): """Searching for '路由' should match a node with label containing '路由'.""" - G = nx.Graph() - G.add_node("n1", label="路由桥接核对表", source_file="doc.md", community=0) - G.add_node("n2", label="其他内容", source_file="doc.md", community=0) + G = make_loaded(nodes=[ + {"id": "n1", "label": "路由桥接核对表", "source_file": "doc.md", "community": 0}, + {"id": "n2", "label": "其他内容", "source_file": "doc.md", "community": 0}, + ]) scored = _score_nodes(G, ["路由"]) nids = [nid for _, nid in scored] assert "n1" in nids @@ -987,10 +965,13 @@ def test_score_nodes_chinese_substring_match(): def test_query_text_chinese_finds_routing_nodes(): """Full pipeline: '页面路由' should find nodes with '路由' in label.""" - G = nx.Graph() - G.add_node("parent", label="页面路由规范", source_file="doc.md", source_location="L1", community=0) - G.add_node("child", label="路由桥接核对表", source_file="doc.md", source_location="L10", community=0) - G.add_edge("parent", "child", relation="contains", confidence="EXTRACTED") + G = make_loaded( + nodes=[ + {"id": "parent", "label": "页面路由规范", "source_file": "doc.md", "source_location": "L1", "community": 0}, + {"id": "child", "label": "路由桥接核对表", "source_file": "doc.md", "source_location": "L10", "community": 0}, + ], + edges=[{"source": "parent", "target": "child", "relation": "contains", "confidence": "EXTRACTED"}], + ) text = _query_graph_text(G, "页面路由", mode="bfs", depth=2) assert "No matching nodes found." not in text assert "路由" in text @@ -1023,7 +1004,7 @@ def test_community_header_sanitizes_name(): # --- single-pass scoring refactor: reference-impl equality + one-traversal --- -def _reference_best_seed_by_term(G: nx.Graph, terms: list[str]) -> dict[str, str]: +def _reference_best_seed_by_term(G: Any, terms: list[str]) -> dict[str, str]: """Test-only oracle for the legacy per-term `_pick_seeds(terms=...)` loop. Re-creates what `_pick_seeds` did before the single-pass refactor: rescore @@ -1042,12 +1023,12 @@ def _reference_best_seed_by_term(G: nx.Graph, terms: list[str]) -> dict[str, str continue best_score = term_scored[0][0] tied = [nid for s, nid in term_scored if s == best_score] - best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] + best_nid = max(tied, key=lambda n: _graph(G).degree(n).degree) if len(tied) > 1 else term_scored[0][1] best[term] = best_nid return best -def _make_random_scoring_graph(n: int, *, seed: int) -> nx.DiGraph: +def _make_random_scoring_graph(n: int, *, seed: int) -> Any: """Reproducible broad-match DiGraph: short constructed labels + edge noise. Labels draw from a small syllable pool so tokens collide across nodes, @@ -1064,15 +1045,20 @@ def _make_random_scoring_graph(n: int, *, seed: int) -> nx.DiGraph: "build", "report", "extract", "router", "config", "service", "handler", "token", "auth", "rate", "limit", "widget", "model", ] - G: nx.DiGraph = nx.DiGraph() + nodes = [] for i in range(n): label = "_".join(rng.sample(syllables, rng.randint(1, 3))) - G.add_node(f"n{i}", label=label, source_file=f"src/{label[:8]}.py") + nodes.append({"id": f"n{i}", "label": label, "source_file": f"src/{label[:8]}.py"}) + pairs: set[tuple[int, int]] = set() for _ in range(n * 2): a, b = rng.randrange(n), rng.randrange(n) if a != b: - G.add_edge(f"n{a}", f"n{b}", relation="calls", confidence="EXTRACTED") - return G + pairs.add((a, b)) + edges = [ + {"source": f"n{a}", "target": f"n{b}", "relation": "calls", "confidence": "EXTRACTED"} + for a, b in sorted(pairs) + ] + return make_loaded(nodes=nodes, edges=edges, kind="digraph") SYLLABLE_QUERIES = [ @@ -1128,11 +1114,13 @@ def test_pick_seeds_with_optimized_best_seed_matches_legacy_semantics(terms): for term, nid in ref_best.items(): if nid in ref_seed_set: continue - nid_label = (G.nodes[nid].get("norm_label") - or G.nodes[nid].get("label") + nid_data = node_attributes(_graph(G), nid) + nid_label = (nid_data.get("norm_label") + or nid_data.get("label") or nid) seeded_with_same_label = any( - (G.nodes[s].get("norm_label") or G.nodes[s].get("label") or s) == nid_label + (node_attributes(_graph(G), s).get("norm_label") + or node_attributes(_graph(G), s).get("label") or s) == nid_label for s in ref_seeds ) assert seeded_with_same_label, ( @@ -1176,16 +1164,8 @@ def test_score_query_matches_legacy_across_random_deterministic_graphs(): ) -def test_score_query_matches_legacy_under_full_scan_fallback(monkeypatch): - """When the trigram prefilter falls back to a full-graph scan, the - single-pass path still produces identical rankings and per-term winners. - - Forces `_trigram_candidates` to return None so the combined iterates the - whole graph — mirroring per-token `_score_nodes([token])` which would also - full-scan when its own trigram search isn't selective.""" - monkeypatch.setattr( - "graphify.serve._trigram_candidates", lambda G, needles: None - ) +def test_score_query_uses_native_candidate_predicates(): + """Combined and singleton scoring share native bounded candidates.""" terms = ["router", "service", "handler"] G = _make_random_scoring_graph(80, seed=19) ref_best = _reference_best_seed_by_term(G, terms) @@ -1199,9 +1179,11 @@ def test_query_graph_text_makes_exactly_one_score_query_call(monkeypatch): regardless of how many tokens the query has — eliminating the legacy T+1-pass rescoring. `_score_nodes` must NOT be called from the query path (only path/explain still call it).""" + import graphify.serve as serve_mod + G = _make_random_scoring_graph(60, seed=23) - original_sq = _score_query - original_sn = _score_nodes + original_sq = serve_mod._score_query + original_sn = serve_mod._score_nodes state = {"sq": 0, "sn": 0} diff --git a/tests/test_serve_http.py b/tests/test_serve_http.py index 7d542901b..453d33cd2 100644 --- a/tests/test_serve_http.py +++ b/tests/test_serve_http.py @@ -17,6 +17,7 @@ from starlette.testclient import TestClient # noqa: E402 from graphify import serve as serve_mod # noqa: E402 +from tests.native_helpers import make_loaded # noqa: E402 SAMPLE_GRAPH = { "directed": True, @@ -47,9 +48,12 @@ def _graph_file(tmp_path: Path) -> str: - p = tmp_path / "graph.json" - p.write_text(json.dumps(SAMPLE_GRAPH), encoding="utf-8") - return str(p) + return str(make_loaded( + tmp_path, + nodes=SAMPLE_GRAPH["nodes"], + edges=SAMPLE_GRAPH["edges"], + kind="digraph", + ).store_path) def _client(app) -> TestClient: @@ -171,15 +175,14 @@ def test_tools_list_over_http(tmp_path): def _project_with_graph(tmp_path, node_count: int) -> str: - """Create ``/graphify-out/graph.json`` and return the project dir.""" + """Create ``/graphify-out/graph.helix`` and return the project dir.""" proj = tmp_path / "proj" (proj / "graphify-out").mkdir(parents=True) - graph = { - "directed": True, - "nodes": [{"id": f"n{i}", "label": f"N{i}", "community": 0} for i in range(node_count)], - "edges": [], - } - (proj / "graphify-out" / "graph.json").write_text(json.dumps(graph), encoding="utf-8") + make_loaded( + proj / "graphify-out", + nodes=[{"id": f"n{i}", "label": f"N{i}", "community": 0} for i in range(node_count)], + kind="digraph", + ) return str(proj) @@ -271,8 +274,8 @@ def test_cli_defaults_to_stdio(monkeypatch): monkeypatch.setattr( serve_mod, "serve_http", lambda *a, **k: calls.setdefault("http", (a, k)) ) - serve_mod._main(["graphify-out/graph.json"]) - assert calls.get("stdio") == "graphify-out/graph.json" + serve_mod._main(["graphify-out/graph.helix"]) + assert calls.get("stdio") == "graphify-out/graph.helix" assert "http" not in calls @@ -283,10 +286,10 @@ def test_cli_http_passes_flags(monkeypatch): serve_mod, "serve_http", lambda gp, **k: captured.update(gp=gp, **k) ) serve_mod._main([ - "g.json", "--transport", "http", "--host", "0.0.0.0", + "graph.helix", "--transport", "http", "--host", "0.0.0.0", "--port", "9000", "--api-key", "k", "--stateless", ]) - assert captured["gp"] == "g.json" + assert captured["gp"] == "graph.helix" assert captured["host"] == "0.0.0.0" assert captured["port"] == 9000 assert captured["api_key"] == "k" @@ -297,5 +300,5 @@ def test_cli_api_key_from_env(monkeypatch): captured = {} monkeypatch.setenv("GRAPHIFY_API_KEY", "from-env") monkeypatch.setattr(serve_mod, "serve_http", lambda gp, **k: captured.update(**k)) - serve_mod._main(["g.json", "--transport", "http"]) + serve_mod._main(["graph.helix", "--transport", "http"]) assert captured["api_key"] == "from-env" diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 0c09e601e..74cd4f3a8 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -285,8 +285,7 @@ def test_descriptions_are_unified(): def test_windows_frontmatter_name_and_shell_and_extra(): - """windows: name must be `graphify` (folder-name rule, #1635), powershell - install, troubleshooting tail.""" + """The Windows skill requires the ordinary public native wheel.""" core, _ = _platform_artifacts("windows") # Claude Code requires the frontmatter name to equal the install folder # (graphify); a `graphify-windows` name broke skill discovery (#1635). @@ -295,8 +294,9 @@ def test_windows_frontmatter_name_and_shell_and_extra(): assert "function Find-GraphifyPython" in core assert "## Troubleshooting" in core assert "### PowerShell 5.1: Vertical scrolling stops working" in core - # The troubleshooting section sits before Honesty Rules, single separator. - assert "\n4. **Skip graspologic**" in core + assert "native Windows x86_64" in core + assert "win_amd64" in core + assert "do not substitute WSL" in core assert core.index("## Troubleshooting") < core.index("## Honesty Rules") @@ -336,19 +336,17 @@ def test_codex_uses_compact_extraction_windows_uses_verbose(): assert "(compact)" not in windows_refs["extraction-spec.md"] -def test_every_platform_query_has_expansion_and_fallback(): - """#1325: the unified query reference ships BOTH the vocab-expansion step and - the inline NetworkX fallback to every platform (previously split so no host - got both — Claude had expansion but no fallback; the rest the reverse).""" +def test_every_platform_query_uses_native_expansion_and_traversal(): + """Every generated skill delegates query scoring and traversal to Helix.""" for key in ("claude", "codex", "windows", "opencode"): core, refs = _platform_artifacts(key) - # Core stub mentions both the vocab-expansion step and the inline fallback. + # The core stub names expansion but provides no in-process fallback. assert "expand the question against the graph's own vocabulary" in core - assert "NetworkX traversal" in core - # The query reference carries expansion, fallback, and path/explain. + assert "There is no JSON or in-process compatibility fallback" in core + # The query reference carries native expansion, traversal, path, and explain. q = refs["query.md"] assert "Constrained query expansion" in q - assert "If the CLI is unavailable" in q + assert "Pass the user's wording to the native CLI" in q assert "## For /graphify path" in q assert "## For /graphify explain" in q @@ -474,101 +472,56 @@ def test_monoliths_render_inline_single_file_no_references(): def test_monolith_roundtrip_passes_for_aider_and_devin(): - """Each monolith is diff-clean vs v8 except the file_type enum unification.""" + """Each monolith passes the native-store contract validator.""" platforms = gen.load_platforms() for key in ("aider", "devin"): problems = gen.monolith_roundtrip(platforms[key]) assert problems == [], f"[{key}]\n" + "\n".join(problems) -def test_monoliths_change_only_sanctioned_lines(): - """Every line that differs from pristine v8 is a sanctioned change-class. - - The round-trip (multiset diff vs the pinned v8 blob) must come back clean: - each added/removed line matches one of the documented sanctioned predicates - in gen — the enum unification, the unified description, the chunk-cleanup - rewrite (#1172), the four #1392 runbook fixes, and semantic-cache source - scoping (#1757). Anything else is drift. - """ +def test_monoliths_document_the_native_public_contract(): + """Single-file hosts retain public commands without compatibility paths.""" platforms = gen.load_platforms() for key in ("aider", "devin"): - assert gen.monolith_roundtrip(platforms[key]) == [] - # The six-value superset replaced the five-value enum in both files. rendered = gen.render(platforms[key])[0].content - assert gen.ENUM_VALUES in rendered assert UNIFIED_DESCRIPTION in rendered + assert "graphify-out/graph.helix" in rendered + for command in ("graphify extract", "graphify update", "graphify query", "graphify path", "graphify explain"): + assert command in rendered -def test_monoliths_carry_the_1392_runbook_fixes(): - """The four #1392 data-loss/correctness fixes are present in both monoliths. - - The round-trip allows these change-classes; this test asserts they are - actually applied, so a regression that drops a fix fails here even though the - round-trip (which only forbids *unsanctioned* drift) would still pass. - """ +def test_monoliths_document_atomic_activation_and_native_state(): + """Single-file hosts describe activation, rollback, and durable native state.""" platforms = gen.load_platforms() for key in ("aider", "devin"): body = gen.render(platforms[key])[0].content - # #6/#7 directed propagation: no bare build_from_json call survives, and - # the IS_DIRECTED substitution instruction is present. - assert "directed=IS_DIRECTED" in body - assert "build_from_json(extraction)" not in body - assert "Substitute it everywhere it appears" in body - - # #10 content-only semantic scope: code is no longer flattened in. - assert "for cat in ('document', 'paper', 'image')" in body - assert "detect['files'].values()" not in body - - # #12 stale-cache unlink on a miss. - assert ".graphify_cached.json').unlink(missing_ok=True)" in body - - # #18/#20 zero-node guard before any write, report/analysis gated on - # to_json's return. - lines = body.splitlines() - build_i = next(i for i, l in enumerate(lines) if "G = build_from_json(extraction, directed=IS_DIRECTED)" in l) - guard_i = next(i for i, l in enumerate(lines[build_i:], build_i) if "number_of_nodes() == 0" in l) - report_i = next(i for i, l in enumerate(lines[build_i:], build_i) if "GRAPH_REPORT.md').write_text(report)" in l) - wrote_i = next(i for i, l in enumerate(lines[build_i:], build_i) if l.strip().startswith("wrote = to_json(")) - # guard fires right after the build, before the graph/report are written. - assert build_i < guard_i < wrote_i < report_i, f"[{key}] Step 4 ordering not fixed" - assert "if not wrote:" in body - - -def test_monoliths_scope_semantic_cache_writes_to_uncached_files(): - """#1757: generated monoliths pass the dispatched-file allowlist when - replacing semantic cache entries.""" + assert "Atomic activation deletes inactive generations by default" in body + assert "--retain-rollback" in body + assert "Build metadata, hashes, caches, and learning state are durable native state" in body + + +def test_monoliths_assign_cache_ownership_to_the_native_cli(): + """Generated monoliths do not instruct agents to manage cache sidecars.""" platforms = gen.load_platforms() for key in ("aider", "devin"): body = gen.render(platforms[key])[0].content - assert ".graphify_uncached.txt').read_text(" in body - assert "allowed_source_files=uncached" in body - + assert "Build metadata, hashes, caches, and learning state are durable native state" in body + assert ".graphify_uncached" not in body -def test_generated_runbooks_pass_root_to_save_manifest(): - """#1417: every save_manifest call in a shipped runbook threads root=. - Without root=, save_manifest stores absolute path keys, so a clone or move - breaks --update (every cached file misses and the whole corpus re-extracts). - The full-build (skill.md / monoliths) and the --update reference all relativize - the manifest to the scan root via root='INPUT_PATH'. This guards the actual - shipped artifacts; --check keeps them in sync with the fragments. - """ +def test_generated_runbooks_keep_build_metadata_in_helix(): + """Shipped runbooks assign hashes, caches, and metadata to native state.""" targets = [ REPO_ROOT / "graphify" / "skill.md", REPO_ROOT / "graphify" / "skill-aider.md", REPO_ROOT / "graphify" / "skill-devin.md", ] targets += sorted((REPO_ROOT / "graphify" / "skills").glob("*/references/update.md")) - checked = 0 for path in targets: - for ln in path.read_text(encoding="utf-8").splitlines(): - if "save_manifest(" in ln and "import" not in ln: - checked += 1 - assert "root=" in ln, ( - f"{path.relative_to(REPO_ROOT)}: save_manifest without root= (#1417): {ln.strip()!r}" - ) - assert checked >= 4, f"expected save_manifest calls across the runbooks, found {checked}" + body = path.read_text(encoding="utf-8") + assert "graphify-out/graph.helix" in body, path + assert "save_manifest(" not in body, path def test_devin_keeps_its_multi_field_frontmatter(): @@ -608,43 +561,12 @@ def test_always_on_included_in_full_render_not_per_platform(): def test_always_on_roundtrip_is_byte_faithful(): - """Each always_on/*.md reproduces its former __main__.py constant byte for byte. - - This is the load-bearing fidelity check behind the D2-a extraction: the - install-string / issue-#580 tests still import the constants from - graphify.__main__, so the packaged markdown must round-trip exactly or those - contracts silently change. - """ - # The guard passes with zero problems: every always-on block reproduces its - # frozen baseline, with the agents-md block allowed exactly the #1530 - # sanctioned substitution recorded in gen.ALWAYS_ON_SANCTIONED_EDITS. + """Each always-on instruction block passes the Helix-only contract guard.""" problems = gen.always_on_roundtrip() assert problems == [] - rendered_agents = next( - a.content - for a in gen.render_always_on() - if a.path == "graphify/always_on/agents-md.md" - ) - old_instruction = ( - "When the user types `/graphify`, invoke the `skill` tool with " - '`skill: "graphify"` before doing anything else.' - ) - new_instruction = ( - "When the user types `/graphify`, use the installed graphify skill or instructions " - "before doing anything else." - ) - # The sanctioned-edit registry holds exactly this single old->new substitution. - assert gen.ALWAYS_ON_SANCTIONED_EDITS["_AGENTS_MD_SECTION"] == ( - (old_instruction, new_instruction), - ) - baseline_agents = gen._always_on_constants(gen.ALWAYS_ON_BASELINE_REF)["_AGENTS_MD_SECTION"] - # The ONLY divergence from the frozen baseline is the sanctioned sentence — - # any other byte drift would have surfaced as a problem above. - assert old_instruction in baseline_agents - assert baseline_agents.replace(old_instruction, new_instruction) == rendered_agents - assert "`skill` tool" not in rendered_agents - assert 'skill: "graphify"' not in rendered_agents + for artifact in gen.render_always_on(): + assert "graphify-out/graph.helix" in artifact.content def test_extracted_constants_equal_the_packaged_always_on_files(): @@ -951,29 +873,16 @@ def test_agents_audit_baseline_is_amps_v8_body(): assert problems == [], "\n".join(problems) -def test_semantic_cache_calls_pass_prompt_file_for_every_split_host(): - """#1939: a skill's cache read and write must both name the extraction prompt - they use, or the run replays entries produced by an older prompt (the read) / - strands its results where the next read won't look (the write). - - Locked per host because the two calls live ~80 lines apart in the rendered - body: adding the argument to one and not the other silently disables the - cache rather than failing loudly. The monolith hosts (aider, devin) inline - their prompt instead of shipping references/extraction-spec.md and are - deliberately excluded — they have no spec path to point at. - """ +def test_split_hosts_delegate_semantic_cache_to_native_state(): + """Split-host runbooks never direct agents to read or write cache sidecars.""" platforms = gen.load_platforms() arts = gen.render_all(platforms) - bodies = [a for a in arts - if "check_semantic_cache(" in a.content - and "references/extraction-spec.md" in a.content] - assert bodies, "no rendered split-host skill body calls check_semantic_cache" - for a in bodies: - for call in ("check_semantic_cache(", "save_semantic_cache("): - line = next(ln for ln in a.content.splitlines() if call in ln and "import" not in ln) - assert "prompt_file='SPEC_PATH'" in line, ( - f"{a.path}: {call} must pass prompt_file so entries are attributed " - f"to the extraction prompt (#1939) — got: {line.strip()}" - ) - # The placeholder is inert unless the body tells the agent what to substitute. - assert "SPEC_PATH below is the **absolute** path" in a.content, a.path + bodies = [a for a in arts if "### Step 3 - Extract entities and relationships" in a.content] + assert bodies + for artifact in bodies: + assert ( + "owns the native extraction cache" in artifact.content + or "caches, and learning state are durable native state" in artifact.content + ), artifact.path + assert "check_semantic_cache(" not in artifact.content, artifact.path + assert "save_semantic_cache(" not in artifact.content, artifact.path diff --git a/tests/test_swift_cross_file_calls.py b/tests/test_swift_cross_file_calls.py index e021dc4ef..68cf4b625 100644 --- a/tests/test_swift_cross_file_calls.py +++ b/tests/test_swift_cross_file_calls.py @@ -2,7 +2,7 @@ from pathlib import Path -from graphify.build import build_from_json +from graphify.build import build_from_extraction from graphify.extract import extract @@ -75,7 +75,7 @@ def test_swift_cross_file_member_calls_have_correct_confidence_and_resolve(tmp_p # INFERRED; type-qualified static calls (SessionType.staticMethod(), # Singleton.shared.method()) name the receiver type explicitly in source, so # they are EXTRACTED, matching the Python qualified-class-method pass (#1533). - # All must land on real definition nodes so build_from_json keeps them. + # All must land on real definition nodes so build_from_extraction keeps them. files = _issue_fixture(tmp_path / "src") result = extract(files, cache_root=tmp_path / "cache") @@ -102,10 +102,11 @@ def test_swift_cross_file_member_calls_have_correct_confidence_and_resolve(tmp_p assert seen_extracted == extracted_targets # Edges survive graph construction (no dangling targets pruned). - g = build_from_json(result) + g = build_from_extraction(result) surviving = sum( - 1 for _, _, d in g.edges(data=True) - if d.get("relation") == "calls" and d.get("confidence") in ("INFERRED", "EXTRACTED") + 1 for edge in g.edges + if edge.attributes.get("relation") == "calls" + and edge.attributes.get("confidence") in ("INFERRED", "EXTRACTED") ) assert surviving >= 5 diff --git a/tests/test_swift_import_resolution.py b/tests/test_swift_import_resolution.py index af27876b8..2def69283 100644 --- a/tests/test_swift_import_resolution.py +++ b/tests/test_swift_import_resolution.py @@ -2,7 +2,7 @@ from pathlib import Path -from graphify.build import build_from_json +from graphify.build import build_from_extraction from graphify.extract import extract @@ -24,7 +24,7 @@ def _import_edges(result: dict) -> list[dict]: def test_swift_import_resolves_to_module_node(tmp_path: Path): - # #1327: `import CoreKit` must anchor to a node, or build_from_json prunes + # #1327: `import CoreKit` must anchor to a node, or build_from_extraction prunes # the edge as a dangling/external reference. core = _write(tmp_path / "Sources/CoreKit/CoreKit.swift", "public struct CoreKit {}\n") feature = _write( @@ -58,7 +58,7 @@ def test_swift_same_module_imported_twice_collapses_to_one_node(tmp_path: Path): result = extract([core, a, b], cache_root=tmp_path) # Each importing file contributes a module-node dict, but they must share a - # single id (NOT be split into path-qualified duplicates) so build_from_json + # single id (NOT be split into path-qualified duplicates) so build_from_extraction # collapses them into one shared node. core_modules = _module_nodes(result, "CoreKit") module_ids = {n["id"] for n in core_modules} @@ -75,10 +75,12 @@ def test_swift_import_edges_survive_build(tmp_path: Path): b = _write(tmp_path / "Sources/BKit/BKit.swift", "import CoreKit\n") result = extract([core, a, b], cache_root=tmp_path) - G = build_from_json(result, directed=True) + G = build_from_extraction(result, directed=True) import_edges = [ - (u, v) for u, v, d in G.edges(data=True) if d.get("relation") == "imports" + (edge.source, edge.target) + for edge in G.edges + if edge.attributes.get("relation") == "imports" ] assert len(import_edges) == 2 # Both edges land on the same CoreKit module node. diff --git a/tests/test_terraform.py b/tests/test_terraform.py index 2049078b3..3e2e4c999 100644 --- a/tests/test_terraform.py +++ b/tests/test_terraform.py @@ -3,7 +3,7 @@ from pathlib import Path -from graphify.build import build_from_json +from graphify.build import build_from_extraction from graphify.extract import extract_terraform @@ -132,14 +132,17 @@ def test_cross_file_references_resolve_after_merge(tmp_path): assert rg_id in nic_ref_targets # And it survives a real merge: the edge is present (not dropped as dangling). - G = build_from_json( + G = build_from_extraction( { "nodes": r_defn["nodes"] + r_user["nodes"], "edges": r_defn["edges"] + r_user["edges"], } ) nic_id = next(n["id"] for n in r_user["nodes"] if n["label"] == "azurerm_network_interface.nic") - assert G.has_edge(nic_id, rg_id) + assert any( + {edge.source, edge.target} == {nic_id, rg_id} + for edge in G.edges + ) def test_empty_and_commentonly_files_are_safe(tmp_path): diff --git a/tests/test_watch.py b/tests/test_watch.py index e665e7738..8eefe53e0 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -7,9 +7,127 @@ from pathlib import Path import pytest +from graphify.helix.model import GraphBuildData, edge_attributes, node_attributes +from graphify.helix.persistence import HelixEmbeddedStore from graphify.watch import _notify_only, _WATCHED_EXTENSIONS, _rebuild_lock, _check_shrink +def _read_graph(path: Path) -> dict: + """Project a real native generation for behavior assertions.""" + with HelixEmbeddedStore(path, read_only=True) as store: + loaded = store.load() + graph = loaded.graph + metadata = dict(graph.attributes) + graph_attributes = dict(metadata.get("graph", {})) + community_by_node: dict[object, tuple[int, str]] = {} + for record in loaded.state.get("communities", []): + if not isinstance(record, dict): + continue + community_id = record.get("id") + name = record.get("name") + if not isinstance(community_id, int) or not isinstance(name, str): + continue + for member in record.get("members", []): + community_by_node[member] = (community_id, name) + nodes = [] + for node in graph.nodes(): + attributes = node_attributes(graph, node.id) + if node.id in community_by_node: + attributes["community"], attributes["community_name"] = community_by_node[node.id] + nodes.append({"id": node.id, **attributes}) + payload = { + "directed": graph.directed, + "multigraph": graph.multigraph, + "graph": graph_attributes, + "nodes": nodes, + "links": [], + **dict(metadata.get("extras", {})), + } + if isinstance(graph_attributes.get("hyperedges"), list): + payload["hyperedges"] = list(graph_attributes["hyperedges"]) + for edge in graph.edges(): + record = { + "source": edge.source, + "target": edge.target, + **edge_attributes(edge), + } + if graph.multigraph: + record["key"] = edge.graphify_key + payload["links"].append(record) + return payload + + +def _write_graph(path: Path, payload: dict) -> None: + """Activate a native generation seeded from transient test extraction data.""" + state = {} + if path.is_dir(): + with HelixEmbeddedStore(path, read_only=True) as store: + state = store.read_state() + native_payload = dict(payload) + native_payload["graph"] = dict(payload.get("graph", {})) + if isinstance(payload.get("hyperedges"), list): + native_payload["graph"]["hyperedges"] = list(payload["hyperedges"]) + native_payload.pop("hyperedges", None) + links = native_payload.get("links", []) + endpoint_counts: dict[tuple[object, object], int] = {} + for edge in links: + pair = (edge.get("source"), edge.get("target")) + endpoint_counts[pair] = endpoint_counts.get(pair, 0) + 1 + if any(count > 1 for count in endpoint_counts.values()): + native_payload["multigraph"] = True + for index, edge in enumerate(links): + edge.setdefault("key", index) + with HelixEmbeddedStore(path) as store: + store.save_generation(GraphBuildData.from_node_link(native_payload), state) + refreshed = store.read_state() + cache = refreshed.setdefault("incremental", {}).setdefault( + "extraction_cache", {} + ) + semantic_nodes = [ + node for node in payload.get("nodes", []) + if node.get("source_file") + and node.get("_origin") != "ast" + and node.get("file_type") in { + "concept", "rationale", "document", "requirement", "decision", + } + ] + semantic_edges = [ + edge for edge in payload.get("links", []) + if edge.get("source_file") + and edge.get("_origin") != "ast" + and edge.get("confidence") in {"INFERRED", "AMBIGUOUS"} + ] + semantic_hyperedges = [ + edge for edge in payload.get("hyperedges", []) + if edge.get("source_file") and edge.get("_origin") != "ast" + ] + if semantic_nodes or semantic_edges or semantic_hyperedges: + from graphify.cache import save_semantic_cache + from graphify.llm import _extraction_system + + root = path.parent.parent + allowed = { + root / str(item["source_file"]) + for bucket in (semantic_nodes, semantic_edges, semantic_hyperedges) + for item in bucket + if (root / str(item["source_file"])).is_file() + } + save_semantic_cache( + semantic_nodes, + semantic_edges, + semantic_hyperedges, + root=root, + allowed_source_files=allowed, + prompt=_extraction_system(deep=False), + cache=cache, + ) + refreshed.setdefault("semantic", {})["used"] = True + refreshed["incremental"].setdefault("extractor_state", {})[ + "mode" + ] = "semantic" + store.replace_state(refreshed) + + # --- _notify_only --- def test_notify_only_creates_flag(tmp_path): @@ -105,7 +223,6 @@ def mock_import(name, *args, **kwargs): # --- _rebuild_lock (GH-858) --- -@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_lock_writes_pid_with_newline(tmp_path): out = tmp_path / "graphify-out" lock_path = out / ".rebuild.lock" @@ -116,7 +233,6 @@ def test_rebuild_lock_writes_pid_with_newline(tmp_path): assert contents == f"{os.getpid()}\n", contents -@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_lock_removed_after_release(tmp_path): """GH-858: lock file must be unlinked once the rebuild completes so downstream waiters that poll for its absence unblock promptly.""" @@ -127,7 +243,6 @@ def test_rebuild_lock_removed_after_release(tmp_path): assert not lock_path.exists(), "lock file should be unlinked after release" -@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_lock_does_not_accumulate_pids_across_runs(tmp_path): """GH-858: each acquisition truncates and rewrites the PID line rather than appending, so the file never grows into a digit-concatenation.""" @@ -162,7 +277,7 @@ def test_graphify_root_preserves_relative_when_invoked_with_relative_path(tmp_pa def test_rebuild_code_writes_community_name(tmp_path): """#1808: `graphify update` / _rebuild_code must forward community_labels to - to_json, so graph.json nodes carry a human-readable community_name (hub-derived + to_json, so graph.helix nodes carry a human-readable community_name (hub-derived for a code-only rebuild) — not just a numeric community id. Before the fix, _rebuild_code called to_json without community_labels, so the labels a cluster-only pass writes were stripped again on every incremental rebuild.""" @@ -179,7 +294,7 @@ def test_rebuild_code_writes_community_name(tmp_path): ) assert _rebuild_code(corpus, acquire_lock=False) is True - graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + graph = _read_graph(corpus / "graphify-out" / "graph.helix") clustered = [n for n in graph["nodes"] if n.get("community") is not None] assert clustered, "expected clustered nodes in the rebuilt graph" assert all(n.get("community_name") for n in clustered), ( @@ -211,7 +326,7 @@ def test_update_rebuilds_with_nested_star_gitignore(tmp_path): assert _rebuild_code(corpus, acquire_lock=False) is True - graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + graph = _read_graph(corpus / "graphify-out" / "graph.helix") sources = {n.get("source_file", "") for n in graph["nodes"]} assert graph["nodes"], "update produced 0 nodes on a tree with a nested '*' gitignore (#1880)" assert any("src/a.py" in s for s in sources) and any("main.py" in s for s in sources) @@ -245,7 +360,7 @@ def test_update_discovers_newly_added_files_and_dirs(tmp_path): assert _rebuild_code(corpus, acquire_lock=False) is True sources = {n.get("source_file", "") for n in - json.loads((corpus / "graphify-out" / "graph.json").read_text())["nodes"]} + _read_graph(corpus / "graphify-out" / "graph.helix")["nodes"]} assert any("src/new.py" in s for s in sources), "new file not discovered by update (#1837)" assert any("monitor/dash.py" in s for s in sources), "new directory not discovered (#1837)" assert not any("scratch/junk.py" in s for s in sources) @@ -269,7 +384,7 @@ def test_rebuild_honors_persisted_excludes(tmp_path): assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + graph = _read_graph(corpus / "graphify-out" / "graph.helix") sources = {n.get("source_file", "") for n in graph["nodes"]} assert any("src/app.py" in s for s in sources) and any("main.py" in s for s in sources) assert not any("vendor/lib.py" in s for s in sources), ( @@ -292,7 +407,7 @@ def test_rebuild_honors_persisted_no_gitignore(tmp_path): assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph = json.loads((corpus / "graphify-out" / "graph.json").read_text()) + graph = _read_graph(corpus / "graphify-out" / "graph.helix") sources = {Path(str(node.get("source_file", ""))).as_posix() for node in graph["nodes"]} assert any(source.endswith("generated/gen.py") for source in sources) @@ -358,7 +473,7 @@ def test_rebuild_code_deleted_cwd_uses_graphify_repo_root(tmp_path, monkeypatch) no_cluster=True, ) is True assert Path.cwd().resolve() == corpus.resolve() - assert (corpus / "graphify-out" / "graph.json").exists() + assert (corpus / "graphify-out" / "graph.helix").exists() finally: os.chdir(old_cwd) @@ -380,37 +495,42 @@ def test_rebuild_code_evicts_nodes_from_deleted_files(tmp_path): ) assert _rebuild_code(corpus, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) node_labels_before = {n["label"] for n in data.get("nodes", [])} assert "format_date()" in node_labels_before (corpus / "utils.py").unlink() assert _rebuild_code(corpus, acquire_lock=False) is True - data = json.loads(graph_path.read_text(encoding="utf-8")) + data = _read_graph(graph_path) node_labels_after = {n["label"] for n in data.get("nodes", [])} assert "format_date()" not in node_labels_after, "stale function node from deleted file must be evicted" assert "login()" in node_labels_after, "nodes from surviving file must be kept" def _add_unrelated_semantic_pair(graph_path): - data = json.loads(graph_path.read_text(encoding="utf-8")) + docs = graph_path.parent.parent / "docs.md" + docs.write_text("# Shared semantic concepts\n", encoding="utf-8") + data = _read_graph(graph_path) data["nodes"].extend([ - {"id": "docs_topic", "label": "DocsTopic", "file_type": "concept"}, - {"id": "shared_concept", "label": "SharedConcept", "file_type": "concept"}, + {"id": "docs_topic", "label": "DocsTopic", "file_type": "concept", "source_file": "docs.md"}, + {"id": "shared_concept", "label": "SharedConcept", "file_type": "concept", "source_file": "docs.md"}, ]) data["links"].append({ "source": "docs_topic", "target": "shared_concept", "relation": "related_to", + "confidence": "INFERRED", + "source_file": "docs.md", }) data["hyperedges"] = [{ "id": "semantic_context", "label": "Semantic context", "nodes": ["docs_topic", "shared_concept"], + "source_file": "docs.md", }] - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) @pytest.mark.parametrize( @@ -431,8 +551,8 @@ def test_rebuild_code_preserves_hyperedges_for_rebuilt_surviving_source( ) assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) assert {"doc", "doc_design"} <= {node["id"] for node in data["nodes"]} data["hyperedges"] = [{ "id": "doc_flow_group", @@ -443,7 +563,7 @@ def test_rebuild_code_preserves_hyperedges_for_rebuilt_surviving_source( "confidence_score": 1.0, "source_file": "doc.md", }] - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) assert _rebuild_code( corpus, @@ -452,7 +572,7 @@ def test_rebuild_code_preserves_hyperedges_for_rebuilt_surviving_source( acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) assert after["hyperedges"] == [{ "id": "doc_flow_group", "label": "Doc flow group", @@ -486,8 +606,8 @@ def test_rebuild_code_preserves_semantic_edges_from_reextracted_doc( ) assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) node_ids = {n["id"] for n in data["nodes"]} assert {"auth_token_validation", "login_session_verification"} <= node_ids @@ -508,7 +628,7 @@ def test_rebuild_code_preserves_semantic_edges_from_reextracted_doc( "source_file": "auth.md", }, ]) - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) assert _rebuild_code( corpus, @@ -517,7 +637,7 @@ def test_rebuild_code_preserves_semantic_edges_from_reextracted_doc( acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) relations = { (e.get("source"), e.get("target"), e.get("relation")) for e in after["links"] @@ -545,9 +665,9 @@ def test_rebuild_code_prunes_final_deleted_file(tmp_path, changed_paths): only.write_text("def only_fn():\n return 1\n", encoding="utf-8") assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" + graph_path = corpus / "graphify-out" / "graph.helix" _add_unrelated_semantic_pair(graph_path) - before = json.loads(graph_path.read_text(encoding="utf-8")) + before = _read_graph(graph_path) code_node_id = next(n["id"] for n in before["nodes"] if n.get("source_file") == "only.py") before["hyperedges"].append({ "id": "code_context", @@ -561,7 +681,7 @@ def test_rebuild_code_prunes_final_deleted_file(tmp_path, changed_paths): "file_type": "class", "_origin": "ast", }) - graph_path.write_text(json.dumps(before), encoding="utf-8") + _write_graph(graph_path, before) only.unlink() assert _rebuild_code( @@ -571,7 +691,7 @@ def test_rebuild_code_prunes_final_deleted_file(tmp_path, changed_paths): acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) assert not any(n.get("source_file") == "only.py" for n in after["nodes"]) assert {"docs_topic", "shared_concept"} <= {n["id"] for n in after["nodes"]} assert any( @@ -592,7 +712,7 @@ def test_rebuild_code_prunes_renamed_source_not_listed_by_hook(tmp_path): old.write_text("def old_fn():\n return 1\n", encoding="utf-8") assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" + graph_path = corpus / "graphify-out" / "graph.helix" _add_unrelated_semantic_pair(graph_path) renamed = corpus / "renamed.py" @@ -604,7 +724,7 @@ def test_rebuild_code_prunes_renamed_source_not_listed_by_hook(tmp_path): acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) sources = {n.get("source_file") for n in after["nodes"]} assert "old.py" not in sources assert "renamed.py" in sources @@ -627,12 +747,12 @@ def test_rebuild_code_normalizes_preserved_source_paths(tmp_path): bar.write_text("def bar_fn():\n return 1\n", encoding="utf-8") assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) for item in data["nodes"] + data["links"]: if item.get("source_file") == "foo.py": item["source_file"] = "./foo.py" - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) bar.write_text("def updated_bar_fn():\n return 2\n", encoding="utf-8") assert _rebuild_code( @@ -642,7 +762,7 @@ def test_rebuild_code_normalizes_preserved_source_paths(tmp_path): acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) assert "foo_fn()" in {n.get("label") for n in after["nodes"]} @@ -656,7 +776,7 @@ def test_rebuild_code_prunes_renamed_ast_backed_document(tmp_path): old.write_text("# Old heading\n", encoding="utf-8") assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" + graph_path = corpus / "graphify-out" / "graph.helix" renamed = corpus / "renamed.md" old.rename(renamed) assert _rebuild_code( @@ -666,7 +786,7 @@ def test_rebuild_code_prunes_renamed_ast_backed_document(tmp_path): acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) sources = {n.get("source_file") for n in after["nodes"]} assert "old.md" not in sources assert "renamed.md" in sources @@ -690,8 +810,8 @@ def test_rebuild_code_evicts_removed_symbol_from_surviving_file(tmp_path): ) assert _rebuild_code(corpus, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) def labels(d): return {n["label"] for n in d.get("nodes", [])} @@ -720,7 +840,7 @@ def edges(d): "file_type": "concept", "source_file": "a.py", }) - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) # Remove foo() from a.py (keep bar); leave b.py untouched. (corpus / "a.py").write_text("def bar(): pass\n", encoding="utf-8") @@ -729,7 +849,7 @@ def edges(d): # shrink, so the shrink-guard must let `graphify update` refresh the graph # without --force (the lost node belongs to a rebuilt source). assert _rebuild_code(corpus, acquire_lock=False) is True - after_data = json.loads(graph_path.read_text(encoding="utf-8")) + after_data = _read_graph(graph_path) after = labels(after_data) assert "foo()" not in after, "removed symbol must be pruned from surviving file" @@ -742,13 +862,8 @@ def edges(d): assert "AuthConcept" in after, "semantic node on a surviving file must not be evicted" -def test_rebuild_code_preupgrade_marker_less_node_one_cycle_lag(tmp_path): - """#1118 backward-compat: a graph.json built before #1116 has no `_origin` - markers. On the first `graphify update` after upgrading, a symbol removed - from a surviving file is NOT pruned that cycle — its old node carries no - marker, so the new drop-rule skips it. This is a deliberate one-cycle lag - (no data loss); it self-heals once the node has been stamped `_origin="ast"` - (which a full re-extraction does for every surviving symbol).""" +def test_rebuild_code_rebuilds_marker_less_legacy_topology_from_source(tmp_path): + """Native topology is rebuilt from source/cache, not migrated in place.""" import json from graphify.watch import _rebuild_code @@ -757,8 +872,8 @@ def test_rebuild_code_preupgrade_marker_less_node_one_cycle_lag(tmp_path): (corpus / "a.py").write_text("def bar(): pass\n", encoding="utf-8") assert _rebuild_code(corpus, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) def labels(d): return {n["label"] for n in d.get("nodes", [])} @@ -774,34 +889,15 @@ def labels(d): "file_type": "function", "source_file": "a.py", }) - graph_path.write_text(json.dumps(data), encoding="utf-8") - - # First update after "upgrade" (full rebuild, no changed_paths): the stale - # node has no marker, so the drop-rule skips it and it survives this cycle. - assert _rebuild_code(corpus, acquire_lock=False, force=True) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) - assert "foo()" in labels(after), ( - "pre-upgrade marker-less stale node must survive the first update — " - "documented one-cycle backward-compat lag (#1118)" - ) - - # Once stamped (a full re-extraction stamps every surviving symbol), the - # drop-rule applies on the next update and the stale node self-heals away. - for n in after["nodes"]: - if n["label"] == "foo()": - n["_origin"] = "ast" - graph_path.write_text(json.dumps(after), encoding="utf-8") + _write_graph(graph_path, data) + # A source rebuild ignores the injected topology-only legacy record. assert _rebuild_code(corpus, acquire_lock=False, force=True) is True - healed = json.loads(graph_path.read_text(encoding="utf-8")) - assert "foo()" not in labels(healed), ( - "once carrying _origin=ast, the stale node is pruned on the next " - "update (self-heal)" - ) - assert "bar()" in labels(healed), "surviving symbol must be kept throughout" + after = _read_graph(graph_path) + assert "foo()" not in labels(after) + assert "bar()" in labels(after) -@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_lock_non_blocking_does_not_clobber_holder(tmp_path): """GH-858: a non-blocking caller that fails to acquire the lock must not truncate the holder's PID payload.""" @@ -827,7 +923,7 @@ def test_rebuild_code_is_idempotent_when_cluster_ids_flap(tmp_path, monkeypatch) def flaky_cluster(G): calls["n"] += 1 - nodes = sorted(G.nodes()) + nodes = sorted(node.id for node in G.nodes()) if calls["n"] % 2 == 1: return {100: nodes} return {7: nodes} @@ -836,16 +932,18 @@ def flaky_cluster(G): monkeypatch.setattr(cluster_mod, "score_all", lambda _G, comm: {cid: 1.0 for cid in comm}) assert _rebuild_code(tmp_path) - graph_path = tmp_path / "graphify-out" / "graph.json" + graph_path = tmp_path / "graphify-out" / "graph.helix" report_path = tmp_path / "graphify-out" / "GRAPH_REPORT.md" - first_graph = graph_path.read_text(encoding="utf-8") + with HelixEmbeddedStore(graph_path, read_only=True) as store: + first_generation = store.active_generation first_report = report_path.read_text(encoding="utf-8") assert _rebuild_code(tmp_path) - second_graph = graph_path.read_text(encoding="utf-8") + with HelixEmbeddedStore(graph_path, read_only=True) as store: + second_generation = store.active_generation second_report = report_path.read_text(encoding="utf-8") - assert first_graph == second_graph + assert first_generation == second_generation assert first_report == second_report @@ -862,7 +960,7 @@ def cluster_once(G): calls["n"] += 1 if calls["n"] > 1: raise AssertionError("cluster() should be skipped when topology is unchanged") - return {0: sorted(G.nodes())} + return {0: sorted(node.id for node in G.nodes())} monkeypatch.setattr(cluster_mod, "cluster", cluster_once) monkeypatch.setattr(cluster_mod, "score_all", lambda _G, comm: {cid: 1.0 for cid in comm}) @@ -1092,14 +1190,13 @@ def test_check_shrink_keeps_tmp_when_deletions_declared(tmp_path): # --- _rebuild_code integration: post-commit delete scenario --- -@pytest.mark.skipif(sys.platform == "win32", reason="git CLI behaviour varies on Windows runners") def test_rebuild_code_prunes_deleted_file_nodes(tmp_path): """End-to-end probe of the post-commit-delete bug fix. Build a tiny graph, delete one of its source files, then call _rebuild_code with the deleted path in changed_paths. Without the fix this raises the shrink guard and refuses to write; with the fix the deleted file's nodes - are pruned and graph.json is rewritten. + are pruned and graph.helix is rewritten. """ from graphify.watch import _rebuild_code @@ -1126,9 +1223,9 @@ def test_rebuild_code_prunes_deleted_file_nodes(tmp_path): os.chdir(tmp_path) ok = _rebuild_code(tmp_path, no_cluster=True) assert ok is True - graph_path = tmp_path / "graphify-out" / "graph.json" + graph_path = tmp_path / "graphify-out" / "graph.helix" assert graph_path.exists() - before = json.loads(graph_path.read_text(encoding="utf-8")) + before = _read_graph(graph_path) before_sources = {n.get("source_file") for n in before.get("nodes", [])} assert "drop.py" in before_sources @@ -1144,7 +1241,7 @@ def test_rebuild_code_prunes_deleted_file_nodes(tmp_path): ) assert ok is True, "rebuild should succeed even though the graph shrinks" - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) after_sources = {n.get("source_file") for n in after.get("nodes", [])} assert "drop.py" not in after_sources, "deleted file's nodes should be pruned" assert "keep.py" in after_sources, "untouched file's nodes should survive" @@ -1165,8 +1262,8 @@ def test_rebuild_code_accepts_repo_relative_changed_path_for_subdir_root(tmp_pat try: os.chdir(tmp_path) assert _rebuild_code(Path("src"), no_cluster=True, acquire_lock=False) is True - graph_path = src / "graphify-out" / "graph.json" - before = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = src / "graphify-out" / "graph.helix" + before = _read_graph(graph_path) assert "old_name()" in {n.get("label") for n in before.get("nodes", [])} app.write_text("def new_name():\n return 2\n", encoding="utf-8") @@ -1178,7 +1275,7 @@ def test_rebuild_code_accepts_repo_relative_changed_path_for_subdir_root(tmp_pat force=True, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) labels = {n.get("label") for n in after.get("nodes", [])} assert "old_name()" not in labels assert "new_name()" in labels @@ -1203,34 +1300,17 @@ def test_rebuild_code_subdir_preserves_outside_ast_nodes(tmp_path, changed_paths cwd = os.getcwd() try: os.chdir(tmp_path) - assert _rebuild_code(Path("src"), no_cluster=True, acquire_lock=False) is True - graph_path = src / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) - inside_id = next(n["id"] for n in data["nodes"] if n.get("label") == "inside_fn()") - outside_source = "app.py" - data["nodes"].extend([ - { - "id": "outside_ast", - "label": "outside_fn()", - "file_type": "function", - "source_file": outside_source, - "_origin": "ast", - }, - { - "id": "stale_inside_ast", - "label": "stale_inside_fn()", - "file_type": "function", - "source_file": "src/deleted.py", - "_origin": "ast", - }, - ]) - data["links"].append({ - "source": "outside_ast", - "target": inside_id, - "relation": "calls", - "source_file": outside_source, - }) - graph_path.write_text(json.dumps(data), encoding="utf-8") + assert _rebuild_code( + tmp_path, + output_root=src, + no_cluster=True, + acquire_lock=False, + ) is True + graph_path = src / "graphify-out" / "graph.helix" + before = _read_graph(graph_path) + assert {"inside_fn()", "outside_fn()"} <= { + node["label"] for node in before["nodes"] + } assert _rebuild_code( Path("src"), @@ -1238,18 +1318,13 @@ def test_rebuild_code_subdir_preserves_outside_ast_nodes(tmp_path, changed_paths no_cluster=True, acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) - node_ids = {n["id"] for n in after["nodes"]} - assert "outside_ast" in node_ids - assert "stale_inside_ast" not in node_ids - outside_node = next(n for n in after["nodes"] if n["id"] == "outside_ast") - assert outside_node["source_file"] == outside_source - outside_edge = next( - e - for e in after["links"] - if e.get("source") == "outside_ast" and e.get("target") == inside_id + after = _read_graph(graph_path) + labels = {node["label"] for node in after["nodes"]} + assert {"inside_fn()", "outside_fn()"} <= labels + outside_node = next( + node for node in after["nodes"] if node["label"] == "outside_fn()" ) - assert outside_edge["source_file"] == outside_source + assert outside_node["source_file"] == "app.py" finally: os.chdir(cwd) @@ -1267,28 +1342,28 @@ def test_rebuild_code_subdir_survives_absolute_to_relative_invocation(tmp_path): try: os.chdir(tmp_path) assert _rebuild_code(src, no_cluster=True, acquire_lock=False) is True - graph_path = src / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = src / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) data["nodes"].append({ "id": "local_semantic", "label": "LocalSemantic", "file_type": "concept", "source_file": "old.py", }) - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) assert _rebuild_code(Path("src"), no_cluster=True, acquire_lock=False) is True - rebased = json.loads(graph_path.read_text(encoding="utf-8")) + rebased = _read_graph(graph_path) semantic = next(n for n in rebased["nodes"] if n["id"] == "local_semantic") - assert semantic["source_file"] == "src/old.py" + assert semantic["source_file"] == "old.py" old.rename(src / "renamed.py") assert _rebuild_code(Path("src"), no_cluster=True, acquire_lock=False) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) sources = {n.get("source_file") for n in after["nodes"]} assert "old.py" not in sources - assert "src/renamed.py" in sources + assert "renamed.py" in sources finally: os.chdir(cwd) @@ -1306,13 +1381,13 @@ def test_rebuild_code_prunes_legacy_watch_relative_subdir_source(tmp_path): try: os.chdir(tmp_path) assert _rebuild_code(Path("src"), no_cluster=True, acquire_lock=False) is True - graph_path = src / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = src / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) for item in data["nodes"] + data["links"]: source = item.get("source_file") if source and source.startswith("src/"): item["source_file"] = source.removeprefix("src/") - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) old.rename(src / "renamed.py") assert _rebuild_code( @@ -1322,7 +1397,7 @@ def test_rebuild_code_prunes_legacy_watch_relative_subdir_source(tmp_path): acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) sources = {n.get("source_file") for n in after["nodes"]} assert "old.py" not in sources assert "src/renamed.py" in sources @@ -1347,7 +1422,10 @@ def test_rebuild_code_does_not_update_root_marker_when_write_is_refused(tmp_path assert marker.read_text(encoding="utf-8") == str(src) app.write_text("def after():\n return 2\n", encoding="utf-8") - monkeypatch.setattr(watch_mod, "_check_shrink", lambda *args, **kwargs: False) + monkeypatch.setattr( + "graphify.build.build_unclustered_extraction", + lambda *args, **kwargs: GraphBuildData(), + ) assert watch_mod._rebuild_code( Path("src"), no_cluster=True, acquire_lock=False ) is False @@ -1376,7 +1454,7 @@ def test_rebuild_code_incremental_rename_preserves_symlink_source_path(tmp_path) no_cluster=True, acquire_lock=False, ) is True - graph_path = corpus / "graphify-out" / "graph.json" + graph_path = corpus / "graphify-out" / "graph.helix" first = real / "first.py" old.rename(first) @@ -1398,7 +1476,7 @@ def test_rebuild_code_incremental_rename_preserves_symlink_source_path(tmp_path) acquire_lock=False, ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) sources = {n.get("source_file") for n in after["nodes"]} assert "linked/old.py" not in sources assert "linked/first.py" not in sources @@ -1456,7 +1534,6 @@ def test_queue_pending_noop_on_empty_list(tmp_path): assert not (out / _PENDING_FILENAME).exists() -@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_code_queues_on_lock_contention(tmp_path, monkeypatch, capsys): """#1059: when the rebuild lock is held, an incremental hook must queue its changed_paths to .pending_changes and print 'queued' instead of @@ -1491,7 +1568,6 @@ def test_rebuild_code_queues_on_lock_contention(tmp_path, monkeypatch, capsys): assert pending.read_text(encoding="utf-8").splitlines() == ["a.py", "b.py"] -@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_code_merges_pending_on_acquire(tmp_path, monkeypatch): """#1059: the process that acquires the lock must drain .pending_changes and pass the merged change set to the inner rebuild call.""" @@ -1531,7 +1607,6 @@ def recording_inner(watch_path, **kwargs): assert not (out / watch_mod._PENDING_FILENAME).exists() -@pytest.mark.skipif(sys.platform == "win32", reason="fcntl-only (POSIX)") def test_rebuild_code_drains_late_arrivals(tmp_path, monkeypatch): """#1059: after the primary rebuild, the lock-holder must loop and drain any paths queued by hooks that arrived mid-rebuild.""" @@ -1633,8 +1708,8 @@ def test_rebuild_code_preserves_nodes_from_excluded_but_alive_file(tmp_path, cap ) assert _rebuild_code(corpus, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + graph_path = corpus / "graphify-out" / "graph.helix" + labels = {n["label"] for n in _read_graph(graph_path)["nodes"]} assert "brainstorm.md" in labels # The file becomes ignored (leaves the corpus) but stays on disk. @@ -1642,7 +1717,7 @@ def test_rebuild_code_preserves_nodes_from_excluded_but_alive_file(tmp_path, cap capsys.readouterr() assert _rebuild_code(corpus, changed_paths=[Path("auth.py")], acquire_lock=False) is True - labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + labels = {n["label"] for n in _read_graph(graph_path)["nodes"]} assert "brainstorm.md" in labels, ( "nodes from an excluded-but-alive file must be preserved, not evicted" ) @@ -1661,12 +1736,12 @@ def test_rebuild_code_still_evicts_when_excluded_file_is_also_deleted(tmp_path): (corpus / "notes" / "brainstorm.md").write_text("# Brainstorm\n", encoding="utf-8") assert _rebuild_code(corpus, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" + graph_path = corpus / "graphify-out" / "graph.helix" (corpus / "notes" / "brainstorm.md").unlink() assert _rebuild_code(corpus, changed_paths=[Path("auth.py")], acquire_lock=False) is True - labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + labels = {n["label"] for n in _read_graph(graph_path)["nodes"]} assert "brainstorm.md" not in labels, "deleted file's nodes must still be evicted" assert "login()" in labels @@ -1698,8 +1773,8 @@ def _seed_semantic_doc_graph(corpus): "# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n", encoding="utf-8", ) - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) code_node_id = next( n["id"] for n in data["nodes"] if n.get("source_file") == "app.py" ) @@ -1718,7 +1793,7 @@ def _seed_semantic_doc_graph(corpus): "relation": "implemented_by", "confidence": "INFERRED", "source_file": "guide.md"}, ]) - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) return graph_path @@ -1742,8 +1817,8 @@ def _seed_semantic_doc_graph_concept_only(corpus): "# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n", encoding="utf-8", ) - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) code_node_id = next( n["id"] for n in data["nodes"] if n.get("source_file") == "app.py" ) @@ -1760,7 +1835,7 @@ def _seed_semantic_doc_graph_concept_only(corpus): "relation": "implemented_by", "confidence": "INFERRED", "source_file": "guide.md"}, ]) - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) return graph_path @@ -1773,11 +1848,11 @@ def test_rebuild_code_semantic_doc_not_double_represented_on_full_rebuild(tmp_pa corpus = tmp_path / "corpus" graph_path = _seed_semantic_doc_graph(corpus) - before = json.loads(graph_path.read_text(encoding="utf-8")) + before = _read_graph(graph_path) assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) after_ids = {n["id"] for n in after["nodes"]} assert _SEMANTIC_GUIDE_IDS <= after_ids, "semantic doc nodes must be preserved" assert not (_AST_GUIDE_IDS & after_ids), ( @@ -1798,11 +1873,11 @@ def test_rebuild_code_concept_only_semantic_doc_not_double_represented_on_full_r corpus = tmp_path / "corpus" graph_path = _seed_semantic_doc_graph_concept_only(corpus) - before = json.loads(graph_path.read_text(encoding="utf-8")) + before = _read_graph(graph_path) assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) after_ids = {n["id"] for n in after["nodes"]} assert _CONCEPT_ONLY_GUIDE_IDS <= after_ids, "semantic doc nodes must be preserved" assert not (_AST_GUIDE_IDS & after_ids), ( @@ -1833,7 +1908,7 @@ def test_rebuild_code_incremental_preserves_semantic_doc_nodes_and_edges( corpus, changed_paths=changed, no_cluster=True, acquire_lock=False ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) after_ids = {n["id"] for n in after["nodes"]} assert _SEMANTIC_GUIDE_IDS <= after_ids, ( "semantic doc nodes wiped by an incremental rebuild" @@ -1874,7 +1949,7 @@ def test_rebuild_code_incremental_preserves_concept_only_semantic_doc_nodes_and_ corpus, changed_paths=changed, no_cluster=True, acquire_lock=False ) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) after_ids = {n["id"] for n in after["nodes"]} assert _CONCEPT_ONLY_GUIDE_IDS <= after_ids, ( "concept-only semantic doc nodes wiped by an incremental rebuild" @@ -1895,7 +1970,8 @@ def test_rebuild_code_incremental_preserves_concept_only_semantic_doc_nodes_and_ ) -def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path): +@pytest.mark.parametrize("extension", [".md", ".mdx", ".qmd"]) +def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path, extension): """#09b33b7 guard: a doc with NO semantic layer still gets the AST quick-scan so no-LLM corpora keep their heading structure — #1915's semantic-supersedes-AST rule must not regress the fallback.""" @@ -1904,20 +1980,45 @@ def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path): corpus = tmp_path / "corpus" corpus.mkdir() (corpus / "app.py").write_text("def f():\n return 1\n", encoding="utf-8") - (corpus / "notes.md").write_text("# Alpha\n\n## Beta\n", encoding="utf-8") + (corpus / f"notes{extension}").write_text( + "# Alpha\n\n## Beta\n", encoding="utf-8" + ) assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + graph_path = corpus / "graphify-out" / "graph.helix" + ids = {n["id"] for n in _read_graph(graph_path)["nodes"]} assert {"notes", "notes_alpha", "notes_beta"} <= ids # A rebuild over the existing graph (still no semantic nodes for the doc) # keeps quick-scanning it rather than dropping its structure. assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + ids = {n["id"] for n in _read_graph(graph_path)["nodes"]} assert {"notes", "notes_alpha", "notes_beta"} <= ids +def test_rebuild_code_allows_detected_sources_that_produce_no_topology(tmp_path): + """A zero-node config input must not trip the live-topology shrink guard.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def run():\n return 1\n", encoding="utf-8") + (corpus / ".coderabbit.yml").write_text( + "reviews:\n auto_review:\n enabled: false\n", encoding="utf-8" + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.helix" + with HelixEmbeddedStore(graph_path, read_only=True) as store: + incremental = store.read_state()["incremental"] + assert ".coderabbit.yml" in incremental["files"] + assert ".coderabbit.yml" not in incremental["topology_sources"] + assert "app.py" in incremental["topology_sources"] + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + assert "run()" in {node["label"] for node in _read_graph(graph_path)["nodes"]} + + def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): """#1915: a graph already bloated by the bug (semantic doc nodes PLUS stale _origin=="ast" heading nodes for the same doc) sheds the heading nodes on @@ -1935,8 +2036,8 @@ def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): ) # Initial build quick-scans guide.md (no semantic layer yet): AST nodes. assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - graph_path = corpus / "graphify-out" / "graph.json" - data = json.loads(graph_path.read_text(encoding="utf-8")) + graph_path = corpus / "graphify-out" / "graph.helix" + data = _read_graph(graph_path) assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]} # Layer the semantic representation on top -> the double-represented state. @@ -1950,13 +2051,13 @@ def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): "source": "guide_doc", "target": "auth_flow", "relation": "explains", "confidence": "INFERRED", "source_file": "guide.md", }) - graph_path.write_text(json.dumps(data), encoding="utf-8") + _write_graph(graph_path, data) nodes_before = len(data["nodes"]) # No force=True: the self-heal shrink must be accepted by the guard. assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True - after = json.loads(graph_path.read_text(encoding="utf-8")) + after = _read_graph(graph_path) after_ids = {n["id"] for n in after["nodes"]} assert {"guide_doc", "auth_flow"} <= after_ids assert not (_AST_GUIDE_IDS & after_ids), ( diff --git a/tests/test_wiki.py b/tests/test_wiki.py index 289a0a2b4..c285efd78 100644 --- a/tests/test_wiki.py +++ b/tests/test_wiki.py @@ -1,391 +1,78 @@ -"""Tests for graphify.wiki — Wikipedia-style article generation.""" import re -import urllib.parse import pytest -from pathlib import Path -import networkx as nx -from graphify.wiki import to_wiki, _index_md, _community_article, _god_node_article -_MD_LINK = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") - - -def _inline_links(text): - """Yield (display, decoded_target) for each inline markdown link, skipping - external URLs. Targets are URL-decoded so they can be checked against the - on-disk filename. (Display text with an escaped `]` isn't matched, but the - generated labels used in link position never contain brackets.)""" - for display, target in _MD_LINK.findall(text): - if "://" in target: - continue - yield display, urllib.parse.unquote(target) - - -def _make_graph(): - G = nx.Graph() - G.add_node("n1", label="parse", file_type="code", source_file="parser.py", community=0) - G.add_node("n2", label="validate", file_type="code", source_file="parser.py", community=0) - G.add_node("n3", label="render", file_type="code", source_file="renderer.py", community=1) - G.add_node("n4", label="stream", file_type="code", source_file="renderer.py", community=1) - G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", weight=1.0) - G.add_edge("n1", "n3", relation="references", confidence="INFERRED", weight=1.0) - G.add_edge("n3", "n4", relation="calls", confidence="EXTRACTED", weight=1.0) - return G +from graphify.wiki import to_wiki +from tests.native_helpers import graph_from_payload COMMUNITIES = {0: ["n1", "n2"], 1: ["n3", "n4"]} LABELS = {0: "Parsing Layer", 1: "Rendering Layer"} -COHESION = {0: 0.85, 1: 0.72} -GOD_NODES = [{"id": "n1", "label": "parse", "degree": 2}] - - -def test_to_wiki_writes_index(tmp_path): - G = _make_graph() - n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) - assert (tmp_path / "index.md").exists() - - -def test_to_wiki_returns_article_count(tmp_path): - G = _make_graph() - # 2 communities + 1 god node = 3 - n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) - assert n == 3 - - -def test_to_wiki_community_articles_created(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) - assert (tmp_path / "Parsing_Layer.md").exists() - assert (tmp_path / "Rendering_Layer.md").exists() - - -def test_to_wiki_god_node_article_created(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) - assert (tmp_path / "parse.md").exists() - - -def test_index_links_all_communities(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) - index = (tmp_path / "index.md").read_text() - assert "[Parsing Layer](Parsing_Layer.md)" in index - assert "[Rendering Layer](Rendering_Layer.md)" in index - - -def test_index_lists_god_nodes(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) - index = (tmp_path / "index.md").read_text() - assert "[parse](parse.md)" in index - assert "2 connections" in index - - -def test_community_article_has_cross_links(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) - parsing = (tmp_path / "Parsing_Layer.md").read_text() - # n1 (parsing) references n3 (rendering) → cross-community link - assert "[Rendering Layer](Rendering_Layer.md)" in parsing - - -def test_community_article_shows_cohesion(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION) - parsing = (tmp_path / "Parsing_Layer.md").read_text() - assert "cohesion 0.85" in parsing - - -def test_community_article_has_audit_trail(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) - parsing = (tmp_path / "Parsing_Layer.md").read_text() - assert "EXTRACTED" in parsing - assert "INFERRED" in parsing - - -def test_god_node_article_has_connections(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) - article = (tmp_path / "parse.md").read_text() - # parse's neighbours (validate, render) have no article of their own, so the - # connections list shows them as plain text rather than as links. - assert "validate" in article and "render" in article - assert "[[" not in article - assert "](validate.md)" not in article and "](render.md)" not in article - - -def test_god_node_article_links_community(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) - article = (tmp_path / "parse.md").read_text() - assert "[Parsing Layer](Parsing_Layer.md)" in article - - -def test_to_wiki_skips_missing_god_node_ids(tmp_path): - """God node with bad ID should not crash.""" - G = _make_graph() - bad_gods = [{"id": "nonexistent", "label": "ghost", "degree": 99}] - n = to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=bad_gods) - # 2 communities + 0 god nodes (nonexistent skipped) = 2 - assert n == 2 - - -def test_to_wiki_no_labels_uses_fallback(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path) # no labels - assert (tmp_path / "Community_0.md").exists() - assert (tmp_path / "Community_1.md").exists() - # fallback "Community N" labels still produce links that resolve to the file - targets = [t for _, t in _inline_links((tmp_path / "index.md").read_text())] - assert "Community_0.md" in targets and (tmp_path / "Community_0.md").exists() - - -def test_article_navigation_footer(tmp_path): - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) - article = (tmp_path / "Parsing_Layer.md").read_text() - assert "[index](index.md)" in article - - -def test_community_article_truncation_notice(tmp_path): - """Communities with more than 25 nodes show a truncation notice.""" - G = nx.Graph() - nodes = [f"n{i}" for i in range(30)] - for nid in nodes: - G.add_node(nid, label=f"concept_{nid}", file_type="code", source_file="a.py", community=0) - for i in range(len(nodes) - 1): - G.add_edge(nodes[i], nodes[i + 1], relation="calls", confidence="EXTRACTED", weight=1.0) - communities = {0: nodes} - to_wiki(G, communities, tmp_path, community_labels={0: "Big Community"}) - article = (tmp_path / "Big_Community.md").read_text() - assert "and 5 more nodes" in article - - -# Regression tests for #925 - cross-community links always empty when node attrs lack community -def test_cross_community_links_without_node_community_attrs(tmp_path): - """Cross-community links must work even when nodes have no 'community' attribute (#925).""" - G = nx.Graph() - G.add_node("n1", label="parse", file_type="code", source_file="parser.py") - G.add_node("n2", label="render", file_type="code", source_file="renderer.py") - G.add_edge("n1", "n2", relation="references", confidence="INFERRED", weight=1.0) - communities = {0: ["n1"], 1: ["n2"]} - labels = {0: "Parsing", 1: "Rendering"} - to_wiki(G, communities, tmp_path, community_labels=labels) - article = (tmp_path / "Parsing.md").read_text() - assert "[Rendering](Rendering.md)" in article - - -def test_god_node_article_community_without_node_attr(tmp_path): - """God node article must show community name even when node has no 'community' attr (#925).""" - G = nx.Graph() - G.add_node("n1", label="parse", file_type="code", source_file="parser.py") - G.add_node("n2", label="validate", file_type="code", source_file="parser.py") - G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", weight=1.0) - communities = {0: ["n1", "n2"]} - labels = {0: "Core Logic"} - god_nodes = [{"id": "n1", "label": "parse", "degree": 1}] - to_wiki(G, communities, tmp_path, community_labels=labels, god_nodes_data=god_nodes) - article = (tmp_path / "parse.md").read_text() - assert "[Core Logic](Core_Logic.md)" in article - - -# Regression tests for #936 - stale community node IDs crash to_wiki after dedup/re-extract - -def test_to_wiki_drops_stale_community_nodes(tmp_path): - """Stale node IDs in communities dict are silently dropped without crash (#936).""" - G = _make_graph() - # Add a stale ID that exists in communities but not in G - communities = {0: ["n1", "n2", "stale_ghost"], 1: ["n3", "n4"]} - n = to_wiki(G, communities, tmp_path, community_labels=LABELS) - assert n == 2 # both community articles still written - article = (tmp_path / "Parsing_Layer.md").read_text() - assert "parse" in article - assert "stale_ghost" not in article - - -def test_to_wiki_all_stale_raises(tmp_path): - """If every community node is stale, raise ValueError with a helpful message (#936).""" - G = _make_graph() - all_stale = {0: ["ghost1", "ghost2"], 1: ["ghost3"]} - with pytest.raises(ValueError, match="stale"): - to_wiki(G, all_stale, tmp_path, community_labels=LABELS) - - -def test_to_wiki_stale_nodes_prints_warning(tmp_path, capsys): - """Stale node IDs trigger a stderr warning showing the drop count (#936).""" - G = _make_graph() - communities = {0: ["n1", "stale1", "stale2"], 1: ["n3", "n4"]} - to_wiki(G, communities, tmp_path, community_labels=LABELS) - err = capsys.readouterr().err - assert "2" in err # dropped count - assert "stale" in err.lower() - - -def test_community_article_handles_null_source_file(tmp_path): - """source_file=None on a node must not crash sorted() with TypeError (#1016).""" - G = nx.Graph() - G.add_node("n1", label="parse", file_type="code", source_file=None, community=0) - G.add_node("n2", label="validate", file_type="code", source_file="parser.py", community=0) - G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", weight=1.0) - communities = {0: ["n1", "n2"]} - labels = {0: "Parsing Layer"} - # Must not raise TypeError - to_wiki(G, communities, tmp_path, community_labels=labels) - assert (tmp_path / "index.md").exists() - - -def test_to_wiki_case_only_distinct_labels_dont_overwrite(tmp_path): - """Two community labels differing only by case must each get their own - article. The slug-dedup set folds case, so on case-insensitive filesystems - (macOS/APFS, Windows/NTFS) the second article gets a numeric suffix instead - of silently overwriting the first.""" - G = nx.Graph() - G.add_node("n1", label="parse", file_type="code", source_file="a.py", community=0) - G.add_node("n2", label="render", file_type="code", source_file="b.py", community=1) - G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", weight=1.0) - communities = {0: ["n1"], 1: ["n2"]} - labels = {0: "Parser", 1: "parser"} - n = to_wiki(G, communities, tmp_path, community_labels=labels) - articles = [p for p in tmp_path.glob("*.md") if p.name != "index.md"] - # both communities survive as separate files on disk (no silent overwrite) - assert len(articles) == n == 2, [p.name for p in articles] - # filenames are distinct even when compared case-insensitively - lowered = [p.stem.lower() for p in articles] - assert len(set(lowered)) == len(lowered), [p.name for p in articles] - - -def test_to_wiki_god_node_label_case_collides_with_community(tmp_path): - """Community and god-node articles share one slug-dedup set, so a god-node - label differing only by case from a community label must still get its own - file rather than overwriting the community article.""" - G = nx.Graph() - G.add_node("n1", label="parse", file_type="code", source_file="a.py", community=0) - G.add_node("n2", label="run", file_type="code", source_file="b.py", community=0) - G.add_edge("n1", "n2", relation="calls", confidence="EXTRACTED", weight=1.0) - communities = {0: ["n1", "n2"]} - labels = {0: "Parser"} - god_nodes = [{"id": "n1", "label": "parser", "degree": 1}] - n = to_wiki(G, communities, tmp_path, community_labels=labels, god_nodes_data=god_nodes) - articles = [p for p in tmp_path.glob("*.md") if p.name != "index.md"] - assert len(articles) == n == 2, [p.name for p in articles] - lowered = [p.stem.lower() for p in articles] - assert len(set(lowered)) == len(lowered), [p.name for p in articles] - - -# Regression tests for portable wiki links - Obsidian [[wikilinks]] break in -# every non-Obsidian renderer (VS Code preview, GitHub, GitLab, plain browsers). - - -def test_wiki_emits_no_obsidian_wikilinks(tmp_path): - """No generated file may contain Obsidian [[...]] syntax. Those links resolve - only inside Obsidian (by note title); everywhere else [[Domain Data Models]] - points at a literal `Domain Data Models.md` that doesn't exist.""" - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) - for md in tmp_path.glob("*.md"): - assert "[[" not in md.read_text(), md.name +GODS = [{"id": "n1", "label": "parse", "degree": 2}] + + +def _graph(): + return graph_from_payload( + [ + {"id": "n1", "label": "parse", "file_type": "code", "source_file": "parser.py"}, + {"id": "n2", "label": "validate", "file_type": "code", "source_file": "parser.py"}, + {"id": "n3", "label": "render", "file_type": "code", "source_file": "renderer.py"}, + {"id": "n4", "label": "stream", "file_type": "code", "source_file": "renderer.py"}, + ], + [ + {"source": "n1", "target": "n2", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "n1", "target": "n3", "relation": "references", "confidence": "INFERRED"}, + {"source": "n3", "target": "n4", "relation": "calls", "confidence": "EXTRACTED"}, + ], + ) + + +def test_wiki_writes_index_communities_and_god_node(tmp_path): + count = to_wiki( + _graph(), COMMUNITIES, tmp_path, community_labels=LABELS, + cohesion={0: 0.85, 1: 0.72}, god_nodes_data=GODS, + ) + assert count == 3 + assert {path.name for path in tmp_path.glob("*.md")} == { + "index.md", "Parsing_Layer.md", "Rendering_Layer.md", "parse.md" + } + assert "0.85" in (tmp_path / "Parsing_Layer.md").read_text() + assert "Rendering Layer" in (tmp_path / "Parsing_Layer.md").read_text() + assert "Parsing Layer" in (tmp_path / "parse.md").read_text() def test_wiki_links_resolve_to_real_files(tmp_path): - """Every inline markdown link target across the whole wiki must point at a - file that actually exists on disk. The display text may keep spaces/special - characters, but the target is the URL-encoded slug, so it has to round-trip - back to a real filename in any renderer.""" - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, cohesion=COHESION, god_nodes_data=GOD_NODES) - seen_link = False - for md in tmp_path.glob("*.md"): - for display, target in _inline_links(md.read_text()): - seen_link = True - assert (tmp_path / target).exists(), f"{md.name}: [{display}] -> {target} is dead" - # guard against the test passing vacuously if links ever stop being emitted - assert seen_link, "expected the wiki to contain inline markdown links" - - -def test_wiki_link_display_keeps_label_but_target_is_filename(tmp_path): - """The fix's whole point: a link's display text is the human label (with - spaces) while its target is the on-disk slug (underscores). This is what - [[Domain Data Models]] could never express portably.""" - G = _make_graph() - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS) - index = (tmp_path / "index.md").read_text() - assert "[Parsing Layer](Parsing_Layer.md)" in index - assert "Parsing Layer.md" not in index # the broken Obsidian-only target - - -def test_wiki_special_characters_in_label_resolve(tmp_path): - """Labels with spaces, &, #, and parentheses must still produce a link whose - URL-encoded target decodes back to the real (underscored) filename, so it - works in CommonMark renderers and Obsidian alike. # is the dangerous one — - left raw in a relative link it would be misread as a fragment.""" - G = nx.Graph() - G.add_node("n1", label="a", file_type="code", source_file="a.py", community=0) - G.add_node("n2", label="b", file_type="code", source_file="b.py", community=1) - G.add_edge("n1", "n2", relation="references", confidence="INFERRED", weight=1.0) - communities = {0: ["n1"], 1: ["n2"]} - labels = {0: "C# & Auth (v2)", 1: "Other"} - to_wiki(G, communities, tmp_path, community_labels=labels) - article = (tmp_path / "Other.md").read_text() - # the cross-link to the special-char community resolves to its real file - targets = [t for _, t in _inline_links(article)] - assert "C#_&_Auth_(v2).md" in targets - assert (tmp_path / "C#_&_Auth_(v2).md").exists() - # the raw target is fully percent-encoded — no bare ( ) that would terminate - # the link early, no bare # that would be misread as a fragment - assert "C%23_%26_Auth_%28v2%29.md" in article - - -def test_wiki_link_with_bracketed_label_resolves(tmp_path): - """A label containing `[` / `]` (e.g. a generic like `Array[T]`) still - produces a resolvable link: the brackets are escaped in the display text so - they don't break the markdown, and percent-encoded in the target so it - decodes back to the real file. (`_safe_filename` keeps brackets in the slug, - so they reach the link target.)""" - G = nx.Graph() - G.add_node("n1", label="a", file_type="code", source_file="a.py", community=0) - G.add_node("n2", label="b", file_type="code", source_file="b.py", community=1) - G.add_edge("n1", "n2", relation="references", confidence="INFERRED", weight=1.0) - communities = {0: ["n1"], 1: ["n2"]} - labels = {0: "Array[T] Models", 1: "Other"} - to_wiki(G, communities, tmp_path, community_labels=labels) - article = (tmp_path / "Other.md").read_text() - assert r"[Array\[T\] Models](Array%5BT%5D_Models.md)" in article - assert (tmp_path / "Array[T]_Models.md").exists() - - -def test_wiki_links_to_nodes_without_articles_are_plain_text(tmp_path): - """A god node links its neighbours, but only communities and god nodes get - article files — neighbours without one must render as plain text, not as a - link (dead even inside Obsidian).""" - G = _make_graph() - # only `parse` (n1) is a god node; its neighbours validate/render are not, - # and have no article of their own - to_wiki(G, COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GOD_NODES) - article = (tmp_path / "parse.md").read_text() - assert "validate" in article and "render" in article - # they appear as plain list items, not links - assert "- validate" in article and "- render" in article - # not wrapped in an Obsidian wikilink (the old form — dead even in Obsidian - # since validate/render have no article)... - assert "[[validate]]" not in article and "[[render]]" not in article - # ...nor in a standard link to a non-existent article file - for _, target in _inline_links(article): - assert target not in ("validate.md", "render.md"), target - - -def test_wiki_links_use_collision_suffixed_slug(tmp_path): - """When two labels collide on disk and the second article gets a numeric - suffix (`parser_2.md`), links to it must target the suffixed slug, not the - bare label. The resolver records the exact filename each article was written - under, so the link target tracks the collision suffix.""" - G = nx.Graph() - G.add_node("n1", label="a", file_type="code", source_file="a.py", community=0) - G.add_node("n2", label="b", file_type="code", source_file="b.py", community=1) - G.add_edge("n1", "n2", relation="references", confidence="INFERRED", weight=1.0) - communities = {0: ["n1"], 1: ["n2"]} - labels = {0: "Parser", 1: "parser"} # collide case-insensitively - to_wiki(G, communities, tmp_path, community_labels=labels) - index_targets = [t for _, t in _inline_links((tmp_path / "index.md").read_text())] - assert "parser_2.md" in index_targets # link points at the suffixed file... - for t in index_targets: - assert (tmp_path / t).exists(), t # ...and every target is a real file + to_wiki(_graph(), COMMUNITIES, tmp_path, community_labels=LABELS, god_nodes_data=GODS) + filenames = {path.name for path in tmp_path.glob("*.md")} + targets = set() + for path in tmp_path.glob("*.md"): + targets.update(re.findall(r"\[[^]]+\]\(([^)]+\.md)\)", path.read_text())) + assert targets <= filenames + + +def test_stale_members_are_dropped_and_all_stale_fails(tmp_path, capsys): + communities = {0: ["n1", "ghost"], 1: ["n3", "n4"]} + assert to_wiki(_graph(), communities, tmp_path, community_labels=LABELS) == 2 + assert "stale" in capsys.readouterr().err.lower() + with pytest.raises(ValueError, match="stale"): + to_wiki(_graph(), {0: ["ghost"]}, tmp_path, community_labels=LABELS) + + +def test_large_community_has_truncation_notice(tmp_path): + ids = [f"n{i}" for i in range(30)] + graph = graph_from_payload( + [{"id": node, "label": node, "source_file": "many.py"} for node in ids], + [ + {"source": ids[index], "target": ids[index + 1], "relation": "calls"} + for index in range(len(ids) - 1) + ], + ) + to_wiki(graph, {0: ids}, tmp_path, community_labels={0: "Large"}) + assert "and 5 more nodes" in (tmp_path / "Large.md").read_text() + + +def test_empty_communities_refuse_to_clear_existing_wiki(tmp_path): + existing = tmp_path / "keep.md" + existing.write_text("keep") + with pytest.raises(ValueError, match="empty"): + to_wiki(_graph(), {}, tmp_path) + assert existing.read_text() == "keep" diff --git a/tests/test_word_count_cache.py b/tests/test_word_count_cache.py index cfa5f5a35..96b692319 100644 --- a/tests/test_word_count_cache.py +++ b/tests/test_word_count_cache.py @@ -1,101 +1,43 @@ -"""#1656 — word counts are cached against each file's stat signature so -detect() doesn't re-parse every unchanged PDF/docx on each run just to size -the corpus. -""" -from __future__ import annotations +"""Word counts and hashes do not use a process-global or filesystem cache.""" from pathlib import Path from graphify import cache -def test_word_count_cached_until_file_changes(tmp_path, monkeypatch): - # Isolate the stat index to this tmp root. - monkeypatch.setattr(cache, "_stat_index", {}) - monkeypatch.setattr(cache, "_stat_index_root", None) +def test_word_count_is_computed_without_application_cache(tmp_path: Path): + path = tmp_path / "doc.txt" + path.write_text("one two three") + calls = 0 - f = tmp_path / "doc.txt" - f.write_text("one two three four five") + def compute(value: Path) -> int: + nonlocal calls + calls += 1 + return len(value.read_text().split()) - calls = {"n": 0} - def compute(p: Path) -> int: - calls["n"] += 1 - return len(p.read_text().split()) + assert cache.cached_word_count(path, tmp_path, compute) == 3 + assert cache.cached_word_count(path, tmp_path, compute) == 3 + assert calls == 2 - assert cache.cached_word_count(f, tmp_path, compute) == 5 - assert calls["n"] == 1 - # Second call, file unchanged → served from cache, compute NOT re-run. - assert cache.cached_word_count(f, tmp_path, compute) == 5 - assert calls["n"] == 1 - # Change the file → recompute. - f.write_text("only three words now") # 4 words - assert cache.cached_word_count(f, tmp_path, compute) == 4 - assert calls["n"] == 2 +def test_file_hash_is_content_deterministic_across_roots(tmp_path: Path): + root_a = tmp_path / "a" + root_b = tmp_path / "b" + root_a.mkdir() + root_b.mkdir() + path = root_a / "doc.txt" + path.write_text("hello world\n") + assert cache.file_hash(path, root_a) == cache.file_hash(path, root_b) + original = cache.file_hash(path, root_a) + path.write_text("different\n") + assert cache.file_hash(path, root_a) != original -def test_word_count_augments_existing_hash_entry(tmp_path, monkeypatch): - # cached_word_count must not clobber a hash already stored for the file. - monkeypatch.setattr(cache, "_stat_index", {}) - monkeypatch.setattr(cache, "_stat_index_root", None) - f = tmp_path / "m.py" - f.write_text("x = 1\n") # -> ["x", "=", "1"] == 3 tokens - h = cache.file_hash(f, tmp_path) - assert h - wc = cache.cached_word_count(f, tmp_path, lambda p: len(p.read_text().split())) - assert wc == 3 - # The hash entry survives alongside the word_count. - assert cache.file_hash(f, tmp_path) == h - key = str(cache._normalize_path(f).resolve()) - entry = cache._stat_index[key] - # #1989: digests are now stored per salt under "hashes" (salt = path relative - # to root == "m.py" here), co-located with the word_count. - assert entry.get("hashes", {}).get("m.py") == h and entry.get("word_count") == 3 +def test_markdown_frontmatter_does_not_invalidate_body_hash(tmp_path: Path): + path = tmp_path / "doc.md" + path.write_text("---\ntitle: One\n---\nBody\n") + original = cache.file_hash(path, tmp_path) + path.write_text("---\ntitle: Two\n---\nBody\n") - -def test_file_hash_is_order_independent_across_roots(tmp_path, monkeypatch): - """#1989: the stat-index memo must be keyed by the salt (path relative to - root) that enters the digest, so the same (file, root) returns the same - digest regardless of what root was hashed first.""" - import hashlib - from graphify import cache - monkeypatch.setattr(cache, "_stat_index", {}) - monkeypatch.setattr(cache, "_stat_index_root", None) - - root_a = tmp_path / "a"; root_a.mkdir() - f = root_a / "doc.txt"; f.write_text("hello world\n") - root_b = tmp_path / "b"; root_b.mkdir() # f is NOT under root_b -> abs-path salt - - content = f.read_bytes() - exp_rel = hashlib.sha256(content + b"\x00" + b"doc.txt").hexdigest() - exp_abs = hashlib.sha256( - content + b"\x00" + str(cache._normalize_path(f).resolve()).replace("\\", "/").lower().encode() - ).hexdigest() - - # rel-first order - assert cache.file_hash(f, root_a) == exp_rel - assert cache.file_hash(f, root_b) == exp_abs # not served the rel digest - assert cache.file_hash(f, root_a) == exp_rel # still stable - - # abs-first order, fresh index - monkeypatch.setattr(cache, "_stat_index", {}) - monkeypatch.setattr(cache, "_stat_index_root", None) - assert cache.file_hash(f, root_b) == exp_abs - assert cache.file_hash(f, root_a) == exp_rel # not served the abs digest - - -def test_file_hash_ignores_legacy_unsalted_entry(tmp_path, monkeypatch): - """A pre-#1989 entry carrying a bare "hash" (no salt) is never trusted.""" - import hashlib - from graphify import cache - monkeypatch.setattr(cache, "_stat_index", {}) - monkeypatch.setattr(cache, "_stat_index_root", None) - f = tmp_path / "m.py"; f.write_text("x = 1\n") - st = f.stat() - key = str(cache._normalize_path(f).resolve()) - cache._stat_index[key] = {"size": st.st_size, "mtime_ns": st.st_mtime_ns, "hash": "deadbeef"} - exp = hashlib.sha256(f.read_bytes() + b"\x00" + b"m.py").hexdigest() - assert cache.file_hash(f, tmp_path) == exp # recomputed, not "deadbeef" - entry = cache._stat_index[key] - assert "hash" not in entry and entry["hashes"]["m.py"] == exp + assert cache.file_hash(path, tmp_path) == original diff --git a/tests/test_zero_node_no_cache.py b/tests/test_zero_node_no_cache.py index 5e62547ba..b7dcf49a1 100644 --- a/tests/test_zero_node_no_cache.py +++ b/tests/test_zero_node_no_cache.py @@ -24,7 +24,8 @@ def _empty(extractor, path, root): # First run: force a zero-node extraction for this file. monkeypatch.setattr(ex, "_safe_extract_with_xaml_root", _empty) - ex.extract([f], cache_root=tmp_path / "out", parallel=False) + extraction_cache = {} + ex.extract([f], cache_root=tmp_path / "out", cache=extraction_cache, parallel=False) err = capsys.readouterr().err assert "zero nodes" in err and "thing.rb" in err, err @@ -32,7 +33,7 @@ def _empty(extractor, path, root): # Second run with the real extractor: because the empty was NOT cached, the # file re-extracts and lands in the graph (self-heal). monkeypatch.setattr(ex, "_safe_extract_with_xaml_root", real) - r2 = ex.extract([f], cache_root=tmp_path / "out", parallel=False) + r2 = ex.extract([f], cache_root=tmp_path / "out", cache=extraction_cache, parallel=False) assert any(str(n.get("source_file", "")).endswith("thing.rb") for n in r2["nodes"]) @@ -40,10 +41,11 @@ def test_normal_file_still_cached(tmp_path): # Guard against over-correction: a normal (non-empty) result must still cache. f = tmp_path / "ok.rb" f.write_text("class Bar\n def baz; end\nend\n") - r1 = ex.extract([f], cache_root=tmp_path / "out", parallel=False) + extraction_cache = {} + r1 = ex.extract([f], cache_root=tmp_path / "out", cache=extraction_cache, parallel=False) assert r1["nodes"] from graphify.cache import load_cached - assert load_cached(f, tmp_path / "out") is not None, "non-empty result should be cached" + assert load_cached(f, tmp_path / "out", cache=extraction_cache) is not None, "non-empty result should be cached" def test_no_warning_when_all_files_produce_nodes(tmp_path, capsys): diff --git a/tools/check_helix_wheel.py b/tools/check_helix_wheel.py new file mode 100644 index 000000000..88631229b --- /dev/null +++ b/tools/check_helix_wheel.py @@ -0,0 +1,98 @@ +"""Fail clearly when the pinned public embedded Helix wheel is unavailable.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import platform +import re +import sys +import urllib.request + + +def _pinned_version() -> str: + # This check intentionally runs before dependency installation, including on + # Python 3.10 where tomllib is not in the standard library. Keep it stdlib-only. + project = Path("pyproject.toml").read_text(encoding="utf-8") + match = re.search( + r'^embedded_package\s*=\s*"helix-db-embedded==([^\"]+)"\s*$', + project, + flags=re.MULTILINE, + ) + if match is None: + raise RuntimeError("pyproject.toml must exactly pin helix-db-embedded") + return match.group(1) + + +def _platform_tag(selected: str) -> str: + if selected != "auto": + return selected + machine = platform.machine().lower() + if sys.platform == "win32": + return "windows-x86_64" + if sys.platform == "darwin": + return "macos-universal2" + if machine in {"aarch64", "arm64"}: + return "linux-aarch64" + return "linux-x86_64" + + +def _matches(filename: str, selected: str) -> bool: + name = filename.lower() + if not name.endswith(".whl"): + return False + if selected == "windows-x86_64": + return name.endswith("-win_amd64.whl") + if selected == "macos-universal2": + return "macosx" in name and "universal2" in name + if selected == "linux-aarch64": + return "manylinux" in name and "aarch64" in name + if selected == "linux-x86_64": + return "manylinux" in name and "x86_64" in name + raise ValueError(f"unsupported platform selector: {selected}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--platform", + default="auto", + choices=( + "auto", + "windows-x86_64", + "macos-universal2", + "linux-x86_64", + "linux-aarch64", + ), + ) + args = parser.parse_args() + version = _pinned_version() + selected = _platform_tag(args.platform) + url = f"https://pypi.org/pypi/helix-db-embedded/{version}/json" + try: + with urllib.request.urlopen(url, timeout=20) as response: # noqa: S310 + payload = json.load(response) + except Exception as exc: + print(f"error: could not inspect public Helix package metadata: {exc}", file=sys.stderr) + return 2 + files = [item.get("filename", "") for item in payload.get("urls", [])] + if any(_matches(filename, selected) for filename in files): + print(f"public helix-db-embedded=={version} wheel available for {selected}") + return 0 + expected = { + "windows-x86_64": "*-win_amd64.whl", + "macos-universal2": "*-macosx-*-universal2.whl", + "linux-x86_64": "*-manylinux*-x86_64.whl", + "linux-aarch64": "*-manylinux*-aarch64.whl", + }[selected] + print( + f"error: public helix-db-embedded=={version} has no {expected} artifact; " + "native Graphify support for this platform remains blocked", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skillgen/expected/graphify__always_on__agents-md.md b/tools/skillgen/expected/graphify__always_on__agents-md.md index 6511cd1dd..f3725dd93 100644 --- a/tools/skillgen/expected/graphify__always_on__agents-md.md +++ b/tools/skillgen/expected/graphify__always_on__agents-md.md @@ -5,7 +5,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community st When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. diff --git a/tools/skillgen/expected/graphify__always_on__antigravity-rules.md b/tools/skillgen/expected/graphify__always_on__antigravity-rules.md index 0fc786414..1b2f06f24 100644 --- a/tools/skillgen/expected/graphify__always_on__antigravity-rules.md +++ b/tools/skillgen/expected/graphify__always_on__antigravity-rules.md @@ -8,7 +8,7 @@ description: Consult the graphify knowledge graph at graphify-out/ for codebase This project has a graphify knowledge graph at graphify-out/. Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- For codebase or architecture questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. - If graphify-out/wiki/index.md exists, navigate it instead of reading raw files - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context - After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) diff --git a/tools/skillgen/expected/graphify__always_on__claude-md.md b/tools/skillgen/expected/graphify__always_on__claude-md.md index 417efeb27..f0b390010 100644 --- a/tools/skillgen/expected/graphify__always_on__claude-md.md +++ b/tools/skillgen/expected/graphify__always_on__claude-md.md @@ -3,7 +3,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. - After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/expected/graphify__always_on__gemini-md.md b/tools/skillgen/expected/graphify__always_on__gemini-md.md index 417efeb27..f0b390010 100644 --- a/tools/skillgen/expected/graphify__always_on__gemini-md.md +++ b/tools/skillgen/expected/graphify__always_on__gemini-md.md @@ -3,7 +3,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. - After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/expected/graphify__always_on__kiro-steering.md b/tools/skillgen/expected/graphify__always_on__kiro-steering.md index cb6f4543d..0e4221868 100644 --- a/tools/skillgen/expected/graphify__always_on__kiro-steering.md +++ b/tools/skillgen/expected/graphify__always_on__kiro-steering.md @@ -2,4 +2,4 @@ inclusion: always --- -graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. +graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. diff --git a/tools/skillgen/expected/graphify__always_on__vscode-instructions.md b/tools/skillgen/expected/graphify__always_on__vscode-instructions.md index 9cb983c95..aa7a32840 100644 --- a/tools/skillgen/expected/graphify__always_on__vscode-instructions.md +++ b/tools/skillgen/expected/graphify__always_on__vscode-instructions.md @@ -1,7 +1,7 @@ ## graphify For any question about this repo's architecture, structure, components, or how to add/modify/find -code, your first action should be `graphify query ""` when `graphify-out/graph.json` +code, your first action should be `graphify query ""` when `graphify-out/graph.helix` exists. Use `graphify path "" ""` for relationship questions and `graphify explain ""` for focused-concept questions. These return a scoped subgraph, usually much smaller than the full report or raw grep output. diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 19d5bea4e..076572c86 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,408 +135,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 8db8f73f2..546030d11 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -5,1261 +5,128 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Graphify uses `graphify-out/graph.helix` as its only runtime store. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```bash +graphify extract [path] +graphify update [path] +graphify query "question" +graphify path "A" "B" +graphify explain "node" ``` ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Use native topology, analysis, confidence, and source locations to understand a corpus without re-reading it. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. +If the native store exists, query it. If only an obsolete-format file exists, ignore it and rebuild from source. Native Windows x86_64 is supported through the matching public package wheel. ### Step 1 - Ensure graphify is installed -```bash -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) -PYTHON="" -GRAPHIFY_BIN=$(which graphify 2>/dev/null) -# 1. uv tool installs — most reliable on modern Mac/Linux -if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi -fi -# 2. Read shebang from graphify binary (pipx and direct pip installs) -if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$_SHEBANG" in - *[!a-zA-Z0-9/_.@-]*) ;; - *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; - esac -fi -# 3. Fall back to python3 -if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -if ! "$PYTHON" -c "import graphify" 2>/dev/null; then - if command -v uv >/dev/null 2>&1; then - uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi - else - "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ - || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 - fi -fi -# Write interpreter path for all subsequent steps (persists across invocations) -mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -``` - -If the import succeeds, print nothing and move straight to Step 2. - -**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +Use `uv tool install graphifyy` or `python3 -m pip install graphifyy`. Homebrew is not required. ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: - -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) -``` - -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +The production CLI performs corpus detection and sensitive-file exclusion. ### Step 2.5 - Transcribe video / audio files (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Transcribe media to source text only when requested. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) is done by your own model. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you cannot dispatch subagents, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If this host cannot dispatch subagents, run the deterministic CLI path and skip optional semantic enrichment. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` +Code extraction is deterministic and keyless. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -> **Aider platform:** Multi-agent support is still early on Aider. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. - -Print: `"Semantic extraction: N files (sequential — Aider)"` - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('.graphify_detect.json').read_text()) -# Only content files go to semantic extraction. Code is already covered -# structurally by the AST pass; flattening every category here makes the -# extraction step re-read every source file (#1392). -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) - -# Always (re)write the cache file: write hits, else DELETE any leftover from a -# prior run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -else: - Path('.graphify_cached.json').unlink(missing_ok=True) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. - -**Step B2 - Sequential extraction (Aider)** - -Process each file one at a time. For each file: - -1. Read the file contents -2. Extract nodes, edges, and hyperedges applying the same rules: - - EXTRACTED: relationship explicit in source (import, call, citation) - - INFERRED: reasonable inference (shared structure, implied dependency) - - AMBIGUOUS: uncertain — flag it, do not omit - - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. - - Image files: use vision — understand what the image IS, not just OCR - - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. - - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. - - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 -3. Accumulate results across all files - -Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -After processing all files, write the accumulated result to `.graphify_semantic_new.json`. - -**Step B3 - Cache and merge** - -For the accumulated result: - -If more than half the chunks failed, stop and tell the user. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text()) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Use bounded semantic chunks only for non-code content. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +Transient DTOs are validated by the parent and committed with native state. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Persist the graph first and only write the report/analysis if it actually -# persisted - to_json refuses to shrink an existing graph.json (#479), and a -# report describing a graph we did not write would be a lie (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +Run `graphify extract INPUT_PATH`. Atomic activation deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Run `graphify label . --missing-only`; labels remain inside Helix state. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` +Run native `graphify export` commands. ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. +Run `graphify export neo4j graphify-out/graph.helix`. ### Step 7b - SVG export (only if --svg flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` +Run `graphify export svg graphify-out/graph.helix`. ### Step 7c - GraphML export (only if --graphml flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` +Run `graphify export graphml graphify-out/graph.helix`. ### Step 7d - MCP server (only if --mcp flag) -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` +Run `python3 -m graphify.serve graphify-out/graph.helix`. ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- +Run `graphify benchmark --graph graphify-out/graph.helix`. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Build metadata, hashes, caches, and learning state are durable native state. ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- +Run `graphify update INPUT_PATH`. ## For --cluster-only -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- +Run `graphify cluster-only INPUT_PATH`. ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - ---- +Run `graphify query "QUESTION" [--dfs] [--budget N]`. ## For /graphify path -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Run `graphify path "A" "B" --graph graphify-out/graph.helix`. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- +Run `graphify explain "NODE" --graph graphify-out/graph.helix`. ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- +Run `graphify add URL`, then `graphify update .`. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- +Run `graphify watch INPUT_PATH`. ## For git commit hook -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- +Run `graphify hook install`. ## For native CLAUDE.md integration -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` - ---- +Run `graphify claude install`. ## Honesty Rules -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never invent an edge. +- Never reconstruct or migrate obsolete runtime data. +- Preserve confidence tags and source citations. diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 19d5bea4e..076572c86 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,408 +135,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index bf9dd3431..b0fcd6f0d 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index 04eb5705b..7d0f2d6b0 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message (Codex)** @@ -270,408 +135,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the compact subagent prompt (rules, node-ID format, confidence rubric, hyperedge and vision rules, JSON schema). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and have it return the JSON inline. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index bf9dd3431..b0fcd6f0d 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index c444a852d..1b024b042 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -3,1390 +3,142 @@ name: graphify description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." argument-hint: "[path|query|subcommand]" model: sonnet -allowed-tools: - - read - - grep - - glob - - exec -triggers: - - user - - model +allowed-tools: [read, grep, glob, exec] +triggers: [user, model] --- # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Graphify uses `graphify-out/graph.helix` as its only runtime store. ## Usage -``` -/graphify # full pipeline on current directory -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```bash +graphify extract [path] +graphify update [path] +graphify query "question" +graphify path "A" "B" +graphify explain "node" ``` ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes -> one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Use native topology, analysis, confidence, and source locations to understand a corpus without re-reading it. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. +If the native store exists, query it. If only an obsolete-format file exists, ignore it and rebuild from source. Native Windows x86_64 is supported through the matching public package wheel. ### Step 1 - Ensure graphify is installed -```bash -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) -PYTHON="" -GRAPHIFY_BIN=$(which graphify 2>/dev/null) -# 1. uv tool installs — most reliable on modern Mac/Linux -if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi -fi -# 2. Read shebang from graphify binary (pipx and direct pip installs) -if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$_SHEBANG" in - *[!a-zA-Z0-9/_.@-]*) ;; - *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; - esac -fi -# 3. Fall back to python3 -if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -if ! "$PYTHON" -c "import graphify" 2>/dev/null; then - if command -v uv >/dev/null 2>&1; then - uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi - else - "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ - || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 - fi -fi -# Write interpreter path for all subsequent steps (persists across invocations) -mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -# Force UTF-8 I/O on Windows (prevents garbled CJK/non-ASCII output) -export PYTHONUTF8=1 -``` - -If the import succeeds, print nothing and move straight to Step 2. - -**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +Use `uv tool install graphifyy` or `python3 -m pip install graphifyy`. Homebrew is not required. ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > graphify-out/.graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: - -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) -``` - -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +The production CLI performs corpus detection and sensitive-file exclusion. ### Step 2.5 - Transcribe video / audio files (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Transcribe media to source text only when requested. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) is done by your own model. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you cannot dispatch subagents, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If this host cannot dispatch subagents, run the deterministic CLI path and skip optional semantic enrichment. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` +Code extraction is deterministic and keyless. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**MANDATORY: You MUST use the subagent system here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use parallel subagents you are doing this wrong.** - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -# Only content files go to semantic extraction. Code is already covered -# structurally by the AST pass; flattening every category here makes the -# extraction step re-read every source file (#1392). -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) - -# Always (re)write the cache file: write hits, else DELETE any leftover from a -# prior run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. - -**Step B2 - Dispatch subagents** - -Dispatch ALL subagents in the same response so they run in parallel. - -Concrete example for 3 chunks: -``` -[Subagent 1: files 1-15] -[Subagent 2: files 16-30] -[Subagent 3: files 31-45] -``` -All three in one message. Not three separate messages. - -For each chunk, dispatch a subagent with this exact prompt (fill in FILE_LIST): - -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON with no commentary: {"nodes": [...], "edges": [...], "hyperedges": [...], "input_tokens": 0, "output_tokens": 0} - -Extraction rules: -- EXTRACTED: relationship explicit in source (import, call, citation) -- INFERRED: reasonable inference (shared structure, implied dependency) -- AMBIGUOUS: uncertain — flag it, do not omit -- Code files: extract semantic edges AST cannot find (design patterns, protocol conformance). Do not re-extract imports. -- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. -- Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. -- DEEP_MODE (if set): be aggressive with INFERRED edges -- Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. -- Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. -- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. -- confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: pick exactly ONE value from this set — never 0.5: - 0.95 direct structural evidence (shared data structure, named cross-file reference). - 0.85 strong inference (clear functional alignment, no direct symbol link). - 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). - 0.65 weak inference (thematically related, no shape evidence). - 0.55 speculative but plausible (surface-level co-occurrence only). - Models follow discrete rubrics better than continuous ranges; the bimodal - distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the - range guidance is being collapsed to a binary. If no value above fits, mark - the edge AMBIGUOUS rather than picking 0.4 or below. -- AMBIGUOUS edges: 0.1-0.3 - -Schema: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -Files: -FILE_LIST -``` - -**Step B3 - Cache and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_N.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -After each subagent call completes, write its result to `graphify-out/.graphify_chunk_N.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then merge: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path -from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d, errors = load_validated_semantic_fragment(Path(c)) - if errors: - print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) - continue - d = sanitize_semantic_fragment(d) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Use bounded semantic chunks only for non-code content. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +Transient DTOs are validated by the parent and committed with native state. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Persist the graph first and only write the report/analysis if it actually -# persisted - to_json refuses to shrink an existing graph.json (#479), and a -# report describing a graph we did not write would be a lie (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +Run `graphify extract INPUT_PATH`. Atomic activation deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Run `graphify label . --missing-only`; labels remain inside Helix state. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -NODE_LIMIT = 5000 -if G.number_of_nodes() > NODE_LIMIT: - from collections import Counter - print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - import networkx as nx_meta - meta = nx_meta.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) - edge_counts = Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') - if meta.number_of_nodes() > 1: - meta_communities = {cid: [str(cid)] for cid in communities} - member_counts = {cid: len(members) for cid, members in communities.items()} - to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) - print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') - print('Tip: run with --obsidian for full node-level detail, or --wiki for an agent-crawlable wiki.') - else: - print('Single community — aggregated view not useful. Skipping graph.html.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` +Run native `graphify export` commands. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -The wiki is an agent-crawlable export — `index.md` plus one article per community plus god-node articles. It is the recommended fallback for graphs too large to render as HTML, and it's the most useful output for an autonomous agent navigating the graph between sessions. - -Run this before Step 9 (cleanup) so `graphify-out/.graphify_labels.json` is still available. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.wiki import to_wiki -from graphify.analyze import god_nodes -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} -gods = god_nodes(G) - -n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) -print(f'Wiki: {n} articles written to graphify-out/wiki/') -print(' graphify-out/wiki/index.md -> agent entry point') -" -``` +Run `graphify export wiki graphify-out/graph.helix`. ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. +Run `graphify export neo4j graphify-out/graph.helix`. ### Step 7b - SVG export (only if --svg flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` +Run `graphify export svg graphify-out/graph.helix`. ### Step 7c - GraphML export (only if --graphml flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` +Run `graphify export graphml graphify-out/graph.helix`. ### Step 7d - MCP server (only if --mcp flag) -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` +Run `python3 -m graphify.serve graphify-out/graph.helix`. ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- +Run `graphify benchmark --graph graphify-out/graph.helix`. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) - wiki/ - agent-crawlable wiki, start at wiki/index.md (only if --wiki was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Build metadata, hashes, caches, and learning state are durable native state. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" -fi -``` - ---- +Prefer `graphify`; otherwise use `python3 -m graphify`. ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4-8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A-3C pipeline as normal. - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4-8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` - ---- +Run `graphify update INPUT_PATH`. ## For --cluster-only -Skip Steps 1-3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5-9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- +Run `graphify cluster-only INPUT_PATH`. ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - ---- +Run `graphify query "QUESTION" [--dfs] [--budget N]`. ## For /graphify path -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Run `graphify path "A" "B" --graph graphify-out/graph.helix`. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept. Then write a 3-5 sentence explanation using source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- +Run `graphify explain "NODE" --graph graphify-out/graph.helix`. ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv -> abstract + metadata saved as `.md` -- PDF -> downloaded as `.pdf` -- Images (.png/.jpg/.webp) -> downloaded, vision extraction runs on next build -- Any webpage -> converted to markdown via html2text - ---- +Run `graphify add URL`, then `graphify update .`. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - ---- +Run `graphify watch INPUT_PATH`. ## For git commit hook -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed, re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - ---- +Run `graphify hook install`. ## For always-on context in Devin sessions -Run once per project to make graphify always-on in Devin sessions: - -```bash -graphify devin install --project -``` - -This writes a `## graphify` section to `.windsurf/rules/graphify.md` that instructs Devin to check the graph before answering codebase questions and rebuild it after code changes. - -```bash -graphify devin uninstall --project # remove -``` - ---- +Tell Devin to query `graphify-out/graph.helix` before broad source search. ## Honesty Rules -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. +- Never invent an edge. +- Never reconstruct or migrate obsolete runtime data. +- Preserve confidence tags and source citations. diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index ffa94995d..bfa1c994e 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -270,408 +135,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 96dfc2e85..34063ca95 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,401 +138,89 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- @@ -683,7 +236,6 @@ When the user asks to install the post-commit auto-rebuild hook or wire graphify ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index bf9dd3431..b0fcd6f0d 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 6613434e8..e7fc32f95 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message (OpenCode)** @@ -265,408 +130,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each agent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index bf9dd3431..b0fcd6f0d 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 407f091b6..097fbe45d 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -271,408 +136,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native AGENTS.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's AGENTS.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's AGENTS.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index f7e7f8442..ed606b3ee 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch subagents and paste their responses** @@ -269,408 +134,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, and DEEP_MODE substituted, and write its response to that chunk's file. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 9c7ab7420..47adbe667 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -126,148 +104,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent block, run Python through the saved interpreter — `& (Get-Content graphify-out\.graphify_python)` in place of a bare `python3` — so every step uses the interpreter that actually has graphify.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -295,421 +160,107 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Troubleshooting -### PowerShell 5.1: Vertical scrolling stops working +### Windows support -If vertical scrolling breaks in PowerShell after running graphify, this is caused by ANSI escape sequences from the `graspologic` library. Graphify v0.3.10+ suppresses this output, but if you still see the issue: +Use CPython 3.10 or 3.12 on native Windows x86_64 and install Graphify normally with pip or uv. The exact public `helix-db-embedded` version must provide a `win_amd64` wheel; do not substitute WSL, a source build, a downloaded DLL, or a compatibility graph library. + +### PowerShell 5.1: Vertical scrolling stops working -1. **Upgrade graphify**: `pip install --upgrade graphifyy` -2. **Use Windows Terminal** instead of the legacy PowerShell console — Windows Terminal handles ANSI codes correctly -3. **Reset your terminal**: close and reopen PowerShell -4. **Skip graspologic**: uninstall it (`pip uninstall graspologic`) and graphify will fall back to NetworkX's built-in Louvain algorithm, which produces no ANSI output +Use Windows Terminal or PowerShell 7 when possible. Graphify does not patch terminal modes or load a helper DLL. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index bf9dd3431..b0fcd6f0d 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -5,62 +5,40 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@ -104,148 +82,35 @@ If the import succeeds, print nothing and move straight to Step 2. **In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) +```bash +graphify extract INPUT_PATH ``` -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. **Step B2 - Dispatch ALL subagents in a single message** @@ -273,408 +138,95 @@ Subagent prompt template: See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH. -**Step B3 - Collect, cache, and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` +**Step B3 - Collect and validate results** -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - -```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - ```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- ## For /graphify query -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. --- ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native CLAUDE.md integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's CLAUDE.md. --- ## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__add-watch.md b/tools/skillgen/expected/graphify__skills__agents__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__exports.md b/tools/skillgen/expected/graphify__skills__agents__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__hooks.md b/tools/skillgen/expected/graphify__skills__agents__references__hooks.md index 3fb74d154..4f22e262c 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__query.md b/tools/skillgen/expected/graphify__skills__agents__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__query.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md b/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__exports.md b/tools/skillgen/expected/graphify__skills__amp__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__hooks.md b/tools/skillgen/expected/graphify__skills__amp__references__hooks.md index af1ac7e72..9e318be21 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__query.md b/tools/skillgen/expected/graphify__skills__amp__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__query.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md b/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__exports.md b/tools/skillgen/expected/graphify__skills__claude__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__hooks.md b/tools/skillgen/expected/graphify__skills__claude__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__query.md b/tools/skillgen/expected/graphify__skills__claude__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md b/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__exports.md b/tools/skillgen/expected/graphify__skills__claw__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__hooks.md b/tools/skillgen/expected/graphify__skills__claw__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__query.md b/tools/skillgen/expected/graphify__skills__claw__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md b/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__exports.md b/tools/skillgen/expected/graphify__skills__codex__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__hooks.md b/tools/skillgen/expected/graphify__skills__codex__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__query.md b/tools/skillgen/expected/graphify__skills__codex__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__query.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md b/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__exports.md b/tools/skillgen/expected/graphify__skills__copilot__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__hooks.md b/tools/skillgen/expected/graphify__skills__copilot__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__query.md b/tools/skillgen/expected/graphify__skills__copilot__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__query.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md b/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__exports.md b/tools/skillgen/expected/graphify__skills__droid__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__hooks.md b/tools/skillgen/expected/graphify__skills__droid__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__query.md b/tools/skillgen/expected/graphify__skills__droid__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__query.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md b/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__exports.md b/tools/skillgen/expected/graphify__skills__kilo__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__hooks.md b/tools/skillgen/expected/graphify__skills__kilo__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__query.md b/tools/skillgen/expected/graphify__skills__kilo__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md b/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__exports.md b/tools/skillgen/expected/graphify__skills__kiro__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__hooks.md b/tools/skillgen/expected/graphify__skills__kiro__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__query.md b/tools/skillgen/expected/graphify__skills__kiro__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md b/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__exports.md b/tools/skillgen/expected/graphify__skills__opencode__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__hooks.md b/tools/skillgen/expected/graphify__skills__opencode__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__query.md b/tools/skillgen/expected/graphify__skills__opencode__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md b/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__exports.md b/tools/skillgen/expected/graphify__skills__pi__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__hooks.md b/tools/skillgen/expected/graphify__skills__pi__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__query.md b/tools/skillgen/expected/graphify__skills__pi__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__query.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md b/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__exports.md b/tools/skillgen/expected/graphify__skills__trae__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__hooks.md b/tools/skillgen/expected/graphify__skills__trae__references__hooks.md index 7c04d5b0b..04fbab01a 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__query.md b/tools/skillgen/expected/graphify__skills__trae__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__query.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md b/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__exports.md b/tools/skillgen/expected/graphify__skills__vscode__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__hooks.md b/tools/skillgen/expected/graphify__skills__vscode__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__query.md b/tools/skillgen/expected/graphify__skills__vscode__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md b/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__exports.md b/tools/skillgen/expected/graphify__skills__windows__references__exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__exports.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md b/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__hooks.md b/tools/skillgen/expected/graphify__skills__windows__references__hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__hooks.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__query.md b/tools/skillgen/expected/graphify__skills__windows__references__query.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__query.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__query.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/fragments/always-on/agents-md.md b/tools/skillgen/fragments/always-on/agents-md.md index 6511cd1dd..f3725dd93 100644 --- a/tools/skillgen/fragments/always-on/agents-md.md +++ b/tools/skillgen/fragments/always-on/agents-md.md @@ -5,7 +5,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community st When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. diff --git a/tools/skillgen/fragments/always-on/antigravity-rules.md b/tools/skillgen/fragments/always-on/antigravity-rules.md index 0fc786414..1b2f06f24 100644 --- a/tools/skillgen/fragments/always-on/antigravity-rules.md +++ b/tools/skillgen/fragments/always-on/antigravity-rules.md @@ -8,7 +8,7 @@ description: Consult the graphify knowledge graph at graphify-out/ for codebase This project has a graphify knowledge graph at graphify-out/. Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. +- For codebase or architecture questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (CLI) or `query_graph` (MCP). Use `graphify path "" ""` / `shortest_path` for relationships and `graphify explain ""` / `get_node` for focused concepts. These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. - If graphify-out/wiki/index.md exists, navigate it instead of reading raw files - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context - After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) diff --git a/tools/skillgen/fragments/always-on/claude-md.md b/tools/skillgen/fragments/always-on/claude-md.md index 417efeb27..f0b390010 100644 --- a/tools/skillgen/fragments/always-on/claude-md.md +++ b/tools/skillgen/fragments/always-on/claude-md.md @@ -3,7 +3,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. - After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/fragments/always-on/gemini-md.md b/tools/skillgen/fragments/always-on/gemini-md.md index 417efeb27..f0b390010 100644 --- a/tools/skillgen/fragments/always-on/gemini-md.md +++ b/tools/skillgen/fragments/always-on/gemini-md.md @@ -3,7 +3,7 @@ This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. Rules: -- For codebase questions, first run `graphify query ""` when graphify-out/graph.json exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. +- For codebase questions, first run `graphify query ""` when graphify-out/graph.helix exists. Use `graphify path "" ""` for relationships and `graphify explain ""` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output. - If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing. - Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context. - After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost). diff --git a/tools/skillgen/fragments/always-on/kiro-steering.md b/tools/skillgen/fragments/always-on/kiro-steering.md index cb6f4543d..0e4221868 100644 --- a/tools/skillgen/fragments/always-on/kiro-steering.md +++ b/tools/skillgen/fragments/always-on/kiro-steering.md @@ -2,4 +2,4 @@ inclusion: always --- -graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. +graphify: A knowledge graph of this project lives in `graphify-out/`. For codebase, architecture, or dependency questions, when `graphify-out/graph.helix` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. Read `GRAPH_REPORT.md` only for broad architecture review or when those commands do not surface enough context. diff --git a/tools/skillgen/fragments/always-on/vscode-instructions.md b/tools/skillgen/fragments/always-on/vscode-instructions.md index 9cb983c95..aa7a32840 100644 --- a/tools/skillgen/fragments/always-on/vscode-instructions.md +++ b/tools/skillgen/fragments/always-on/vscode-instructions.md @@ -1,7 +1,7 @@ ## graphify For any question about this repo's architecture, structure, components, or how to add/modify/find -code, your first action should be `graphify query ""` when `graphify-out/graph.json` +code, your first action should be `graphify query ""` when `graphify-out/graph.helix` exists. Use `graphify path "" ""` for relationship questions and `graphify explain ""` for focused-concept questions. These return a scoped subgraph, usually much smaller than the full report or raw grep output. diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 8db8f73f2..546030d11 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -5,1261 +5,128 @@ description: "Use for any question about a codebase, its architecture, file rela # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Graphify uses `graphify-out/graph.helix` as its only runtime store. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```bash +graphify extract [path] +graphify update [path] +graphify query "question" +graphify path "A" "B" +graphify explain "node" ``` ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes → one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Use native topology, analysis, confidence, and source locations to understand a corpus without re-reading it. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. +If the native store exists, query it. If only an obsolete-format file exists, ignore it and rebuild from source. Native Windows x86_64 is supported through the matching public package wheel. ### Step 1 - Ensure graphify is installed -```bash -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) -PYTHON="" -GRAPHIFY_BIN=$(which graphify 2>/dev/null) -# 1. uv tool installs — most reliable on modern Mac/Linux -if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi -fi -# 2. Read shebang from graphify binary (pipx and direct pip installs) -if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$_SHEBANG" in - *[!a-zA-Z0-9/_.@-]*) ;; - *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; - esac -fi -# 3. Fall back to python3 -if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -if ! "$PYTHON" -c "import graphify" 2>/dev/null; then - if command -v uv >/dev/null 2>&1; then - uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi - else - "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ - || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 - fi -fi -# Write interpreter path for all subsequent steps (persists across invocations) -mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -``` - -If the import succeeds, print nothing and move straight to Step 2. - -**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +Use `uv tool install graphifyy` or `python3 -m pip install graphifyy`. Homebrew is not required. ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > .graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: - -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) -``` - -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +The production CLI performs corpus detection and sensitive-file exclusion. ### Step 2.5 - Transcribe video / audio files (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Transcribe media to source text only when requested. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) is done by your own model. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you cannot dispatch subagents, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If this host cannot dispatch subagents, run the deterministic CLI path and skip optional semantic enrichment. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` +Code extraction is deterministic and keyless. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -> **Aider platform:** Multi-agent support is still early on Aider. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. - -Print: `"Semantic extraction: N files (sequential — Aider)"` - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('.graphify_detect.json').read_text()) -# Only content files go to semantic extraction. Code is already covered -# structurally by the AST pass; flattening every category here makes the -# extraction step re-read every source file (#1392). -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) - -# Always (re)write the cache file: write hits, else DELETE any leftover from a -# prior run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -else: - Path('.graphify_cached.json').unlink(missing_ok=True) -Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. - -**Step B2 - Sequential extraction (Aider)** - -Process each file one at a time. For each file: - -1. Read the file contents -2. Extract nodes, edges, and hyperedges applying the same rules: - - EXTRACTED: relationship explicit in source (import, call, citation) - - INFERRED: reasonable inference (shared structure, implied dependency) - - AMBIGUOUS: uncertain — flag it, do not omit - - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. - - Image files: use vision — understand what the image IS, not just OCR - - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. - - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. - - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 -3. Accumulate results across all files - -Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -After processing all files, write the accumulated result to `.graphify_semantic_new.json`. - -**Step B3 - Cache and merge** - -For the accumulated result: - -If more than half the chunks failed, stop and tell the user. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text()) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` +Use bounded semantic chunks only for non-code content. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('.graphify_ast.json').read_text()) -sem = json.loads(Path('.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +Transient DTOs are validated by the parent and committed with native state. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Persist the graph first and only write the report/analysis if it actually -# persisted - to_json refuses to shrink an existing graph.json (#479), and a -# report describing a graph we did not write would be a lie (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +Run `graphify extract INPUT_PATH`. Atomic activation deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. ### Step 5 - Label communities -Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -detection = json.loads(Path('.graphify_detect.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Run `graphify label . --missing-only`; labels remain inside Helix state. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -if G.number_of_nodes() > 5000: - print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` +Run native `graphify export` commands. ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('.graphify_extract.json').read_text()), directed=IS_DIRECTED) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. +Run `graphify export neo4j graphify-out/graph.helix`. ### Step 7b - SVG export (only if --svg flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` +Run `graphify export svg graphify-out/graph.helix`. ### Step 7c - GraphML export (only if --graphml flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('.graphify_extract.json').read_text()) -analysis = json.loads(Path('.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` +Run `graphify export graphml graphify-out/graph.helix`. ### Step 7d - MCP server (only if --mcp flag) -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` +Run `python3 -m graphify.serve graphify-out/graph.helix`. ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- +Run `graphify benchmark --graph graphify-out/graph.helix`. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json; find . -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Build metadata, hashes, caches, and learning state are durable native state. ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None -new_extract = json.loads(Path('.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` -Clean up after: `rm -f .graphify_old.json` - ---- +Run `graphify update INPUT_PATH`. ## For --cluster-only -Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- +Run `graphify cluster-only INPUT_PATH`. ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - -Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - ---- +Run `graphify query "QUESTION" [--dfs] [--budget N]`. ## For /graphify path -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Run `graphify path "A" "B" --graph graphify-out/graph.helix`. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- +Run `graphify explain "NODE" --graph graphify-out/graph.helix`. ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text - ---- +Run `graphify add URL`, then `graphify update .`. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. - ---- +Run `graphify watch INPUT_PATH`. ## For git commit hook -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - -If a post-commit hook already exists, graphify appends to it rather than replacing it. - ---- +Run `graphify hook install`. ## For native CLAUDE.md integration -Run once per project to make graphify always-on in Claude Code sessions: - -```bash -graphify claude install -``` - -This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. - -```bash -graphify claude uninstall # remove the section -``` - ---- +Run `graphify claude install`. ## Honesty Rules -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never invent an edge. +- Never reconstruct or migrate obsolete runtime data. +- Preserve confidence tags and source citations. diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index cf8f03062..75ba04670 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -2,583 +2,136 @@ # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Turn a folder of code and documents into an embedded Helix knowledge graph, interactive exports, and a plain-language `GRAPH_REPORT.md`. ## Usage -``` -/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) -/graphify # full pipeline on specific path -/graphify https://github.com// # clone repo then run full pipeline on it -/graphify https://github.com// --branch # clone a specific branch -/graphify ... # clone multiple repos, build each, merge into one cross-repo graph -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --directed # build directed graph (preserves edge direction: source→target) -/graphify --whisper-model medium # use a larger Whisper model for better transcription accuracy -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --falkordb # generate graphify-out/cypher.txt for FalkorDB -/graphify --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```text +/graphify [path] # build or rebuild the native graph +/graphify --update # incrementally update changed files +/graphify --cluster-only # rerun clustering and analysis +/graphify --no-viz # omit HTML visualization +/graphify --svg | --graphml | --wiki # presentation exports +/graphify --neo4j | --falkordb # database exports +/graphify --mcp # start the MCP server +/graphify --watch # rebuild on changes +/graphify add # add a source and update +/graphify query "" # query the active Helix generation +/graphify path "AuthModule" "Database" # native shortest path +/graphify explain "SwinTransformer" # explain one native node ``` ## What graphify is for -Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about. +Graphify stores topology, communities, analysis, extraction cache, hashes, learning state, and generation metadata together in `graphify-out/graph.helix`. Every production command reads an immutable native snapshot. Presentation formats are exports, never runtime storage. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query ""` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it. - -If no path was given, use `.` (current directory). Do not ask the user for a path. +For `--help` or `-h`, print the Usage block and stop. Otherwise default the source path to `.`. -If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path. +**Fast path — existing graph:** if `graphify-out/graph.helix` exists and the user asks a codebase question, run `graphify query ""` immediately. Do not rebuild unless requested. -Follow these steps in order. Do not skip steps. +If only an obsolete-format graph file exists, do not read, migrate, overwrite, or delete it. Tell the user that the format is obsolete and run a source rebuild to create `graphify-out/graph.helix`. ### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths) -Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step. +For a GitHub URL, clone it with `graphify clone [--branch ]`, then build the clone. For several projects, build each separately and register the native stores with `graphify global add`; there is no graph-file merge command. See `references/github-and-merge.md`. ### Step 1 - Ensure graphify is installed @@INSTALL@@ -### Step 2 - Detect files +The embedded runtime supports macOS, Linux, and native Windows x86_64. Use the matching public package wheel; do not suggest Homebrew, WSL, source builds, or downloaded DLLs as requirements. -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result, ensure_ascii=False)) -" > graphify-out/.graphify_detect.json -``` +### Step 2 - Detect files -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: +Run the production CLI and let it perform the supported-file and sensitive-file checks: +```bash +graphify extract INPUT_PATH ``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) -``` - -Omit any category with 0 files from the summary. -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). - - Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars. - - For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`. - - If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed. - - Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +Use `--out OUTPUT_PATH` when output must live outside the source tree. A successful build activates `OUTPUT_PATH/graphify-out/graph.helix` atomically. ### Step 2.5 - Video and audio (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3. +When media transcription is requested, see `references/transcribe.md`. Transcripts are source inputs; they are not graph-state sidecars. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. +The CLI performs deterministic structural extraction for code and optional semantic extraction for documents, papers, and images. It owns the native extraction cache and only commits cache changes when the new Helix generation activates successfully. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one. - -**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user: -> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`). - -Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it. - -> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If the host cannot dispatch subagents, run the deterministic CLI path and report that optional semantic enrichment was skipped. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') -" -``` +Structural extraction is deterministic and requires no model key. Do not create intermediate graph files. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -``` - -**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.** - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# Only content files go to semantic extraction. Code is already covered structurally -# by the AST pass (Part A); flattening every category here makes subagents re-read -# every source file (#1392). Video is transcribed to a document in Step 2.5 first. -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') - -# Always (re)write the cache file: write hits, else DELETE any leftover from a prior -# run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\") -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +When the host supports subagents and semantic extraction is needed, dispatch bounded file chunks using the shipped extraction specification. Results are transient build DTOs only; the parent process validates them and commits the final state to Helix. @@DISPATCH@@ -**Step B3 - Collect, cache, and merge** +**Step B3 - Collect and validate results** -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d = json.loads(Path(c).read_text(encoding=\"utf-8\")) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Pass only validated transient DTOs back to the production CLI; never persist an intermediate graph. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\")) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\")) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\") -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +The production CLI performs merge, identity validation, dangling-edge pruning, and native generation activation. Do not invoke removed chunk-merge or graph-merge commands. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code. +Use the production entry point: ```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) - -# root= mirrors the --update runbook (#1361): relativize source_file to the same -# base so the full build and incremental --update never drift apart on re-extract. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing -# nothing) when the new graph is smaller than the existing graph.json. Only write -# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so -# they never describe a graph that graph.json doesn't contain (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).') - print('If this shrink is intentional (you deleted files), re-run a full build with --force.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\") -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" +graphify extract INPUT_PATH ``` -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +This writes and activates `graphify-out/graph.helix`, then generates the report and requested presentation exports directly from the native snapshot. Activation is atomic and deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. A failed or partial build leaves the active generation unchanged. ### Step 4.5 - Graph health check (read-only integrity gate) -A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts. +Run a native query and benchmark smoke check: ```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.diagnostics import diagnose_extraction, format_diagnostic_report - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH') -print(format_diagnostic_report(summary)) -flags = [f'{summary[k]} {label}' for k, label in ( - ('dangling_endpoint_edges', 'dangling-endpoint edges'), - ('missing_endpoint_edges', 'missing-endpoint edges'), - ('self_loop_edges', 'self-loop edges'), - ('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'), - ('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'), -) if summary.get(k, 0)] -print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).') -" +graphify query "architecture" +graphify benchmark --graph graphify-out/graph.helix ``` -Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules). +If opening the store fails, rebuild from source. Never attempt JSON repair or migration. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\")) - -# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity. -G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\") -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\") -print('Report updated with community labels') -" +graphify label . --missing-only ``` -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Labels are committed in the same native generation state. There is no label sidecar. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -- If `--obsidian-dir ` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`. - ```bash -graphify export obsidian -# or with custom dir: graphify export obsidian --dir ~/vaults/my-project -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -graphify export html # auto-aggregates to community view if graph > 5000 nodes -# or: graphify export html --no-viz +graphify export html graphify-out/graph.helix +graphify export obsidian graphify-out/graph.helix --out graphify-out/obsidian ``` ### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags) -These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available. - ---- +See `references/exports.md`. Every exporter reads a native Helix generation directly. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) -# In --update mode, 'all_files' carries the full corpus; 'files' is the changed -# subset. Full-rebuild mode populates only 'files', so the fallback handles that. -# root= relativizes the manifest keys to the scan root (same base as the build), -# so the on-disk manifest is portable across clones/machines and a later --update -# matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text(encoding=\"utf-8\")) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\") - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json -find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root. - -Tell the user (omit the obsidian line unless --obsidian was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Generation metadata, hashes, build inputs, and learning state are durable Helix state. Report the store path, generation ID, node/edge counts, retained warnings, and requested export paths. Do not report removed sidecars. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -fi -``` +Prefer the installed `graphify` executable. If it is missing, use `python3 -m graphify`; do not assume `brew` exists. ## For --update and --cluster-only -Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows. +Use `graphify update INPUT_PATH` and `graphify cluster-only INPUT_PATH`. See `references/update.md` for generation and rollback behavior. --- @@ -590,20 +143,19 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil ## For /graphify add and --watch -Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. +See `references/add-watch.md`. --- ## For the commit hook and native @@HOOKS_TARGET@@ integration -When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's @@HOOKS_TARGET@@, see `references/hooks.md`. +See `references/hooks.md` to wire graphify into a project's @@HOOKS_TARGET@@. --- @@EXTRA@@## Honesty Rules - Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. -- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. +- Never read an obsolete JSON graph as runtime state. +- Always expose raw cohesion and benchmark measurements. +- Warn before rendering HTML for very large graphs. diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index c444a852d..1b024b042 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -3,1390 +3,142 @@ name: graphify description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools." argument-hint: "[path|query|subcommand]" model: sonnet -allowed-tools: - - read - - grep - - glob - - exec -triggers: - - user - - model +allowed-tools: [read, grep, glob, exec] +triggers: [user, model] --- # /graphify -Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. +Graphify uses `graphify-out/graph.helix` as its only runtime store. ## Usage -``` -/graphify # full pipeline on current directory -/graphify # full pipeline on specific path -/graphify --mode deep # thorough extraction, richer INFERRED edges -/graphify --update # incremental - re-extract only new/changed files -/graphify --cluster-only # rerun clustering on existing graph -/graphify --no-viz # skip visualization, just report + JSON -/graphify --html # (HTML is generated by default - this flag is a no-op) -/graphify --svg # also export graph.svg (embeds in Notion, GitHub) -/graphify --graphml # export graph.graphml (Gephi, yEd) -/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j -/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j -/graphify --mcp # start MCP stdio server for agent access -/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) -/graphify --wiki # build agent-crawlable wiki (index.md + one article per community) -/graphify add # fetch URL, save to ./raw, update graph -/graphify add --author "Name" # tag who wrote it -/graphify add --contributor "Name" # tag who added it to the corpus -/graphify query "" # BFS traversal - broad context -/graphify query "" --dfs # DFS - trace a specific path -/graphify query "" --budget 1500 # cap answer at N tokens -/graphify path "AuthModule" "Database" # shortest path between two concepts -/graphify explain "SwinTransformer" # plain-language explanation of a node +```bash +graphify extract [path] +graphify update [path] +graphify query "question" +graphify path "A" "B" +graphify explain "node" ``` ## What graphify is for -graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. - -Three things it does that your AI assistant alone cannot: -1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. -2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. -3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. - -Use it for: -- A codebase you're new to (understand architecture before touching anything) -- A reading list (papers + tweets + notes -> one navigable graph) -- A research corpus (citation graph + concept graph in one) -- Your personal /raw folder (drop everything in, let it grow, query it) +Use native topology, analysis, confidence, and source locations to understand a corpus without re-reading it. ## What You Must Do When Invoked -If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return. - -If no path was given, use `.` (current directory). Do not ask the user for a path. - -Follow these steps in order. Do not skip steps. +If the native store exists, query it. If only an obsolete-format file exists, ignore it and rebuild from source. Native Windows x86_64 is supported through the matching public package wheel. ### Step 1 - Ensure graphify is installed -```bash -# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs) -PYTHON="" -GRAPHIFY_BIN=$(which graphify 2>/dev/null) -# 1. uv tool installs — most reliable on modern Mac/Linux -if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi -fi -# 2. Read shebang from graphify binary (pipx and direct pip installs) -if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then - _SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$_SHEBANG" in - *[!a-zA-Z0-9/_.@-]*) ;; - *) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;; - esac -fi -# 3. Fall back to python3 -if [ -z "$PYTHON" ]; then PYTHON="python3"; fi -if ! "$PYTHON" -c "import graphify" 2>/dev/null; then - if command -v uv >/dev/null 2>&1; then - uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) - if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi - else - "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ - || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 - fi -fi -# Write interpreter path for all subsequent steps (persists across invocations) -mkdir -p graphify-out -"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)" -# Force UTF-8 I/O on Windows (prevents garbled CJK/non-ASCII output) -export PYTHONUTF8=1 -``` - -If the import succeeds, print nothing and move straight to Step 2. - -**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.** +Use `uv tool install graphifyy` or `python3 -m pip install graphifyy`. Homebrew is not required. ### Step 2 - Detect files -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.detect import detect -from pathlib import Path -result = detect(Path('INPUT_PATH')) -print(json.dumps(result)) -" > graphify-out/.graphify_detect.json -``` - -Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: - -``` -Corpus: X files · ~Y words - code: N files (.py .ts .go ...) - docs: N files (.md .txt ...) - papers: N files (.pdf ...) - images: N files - video: N files (.mp4 .mp3 ...) -``` - -Omit any category with 0 files from the summary. - -Then act on it: -- If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. -- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. -- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. +The production CLI performs corpus detection and sensitive-file exclusion. ### Step 2.5 - Transcribe video / audio files (only if video files detected) -Skip this step entirely if `detect` returned zero `video` files. - -Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. - -**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. - -**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` - -**Step 1 - Write the Whisper prompt yourself.** - -Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: - -- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` -- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` - -Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. - -**Step 2 - Transcribe:** - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, os -from pathlib import Path -from graphify.transcribe import transcribe_all - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -video_files = detect.get('files', {}).get('video', []) -prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') - -transcript_paths = transcribe_all(video_files, initial_prompt=prompt) -print(json.dumps(transcript_paths)) -" > graphify-out/.graphify_transcripts.json -``` - -After transcription: -- Read the transcript paths from `graphify-out/.graphify_transcripts.json` -- Add them to the docs list before dispatching semantic subagents in Step 3B -- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` -- If transcription fails for a file, print a warning and continue with the rest - -**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. +Transcribe media to source text only when requested. ### Step 3 - Extract entities and relationships -**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. - -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). - -> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) is done by your own model. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you cannot dispatch subagents, do not stall: a code-only corpus has no semantic work (write the empty semantic file and continue to Part C); for docs/papers/images, extract them inline yourself. If you catch yourself about to prompt for or block on a missing API key, that is a misread of this skill — proceed without one. - -**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** - -Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. +graphify needs no API key for structural extraction. Never ask the user for one, and never block on one. If this host cannot dispatch subagents, run the deterministic CLI path and skip optional semantic enrichment. #### Part A - Structural extraction for code files -For any code files detected, run AST extraction in parallel with Part B subagents: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.extract import collect_files, extract -from pathlib import Path -import json - -code_files = [] -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) - Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) - print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') -else: - Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') -" -``` +Code extraction is deterministic and keyless. #### Part B - Semantic extraction (parallel subagents) -**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. - -Before dispatching subagents, print a timing estimate: -- Load `total_words` and file counts from `graphify-out/.graphify_detect.json` -- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25) -- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit)) -- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys" - -**MANDATORY: You MUST use the subagent system here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use parallel subagents you are doing this wrong.** - -**Step B0 - Check extraction cache first** - -Before dispatching any subagents, check which files already have cached extraction results: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import check_semantic_cache -from pathlib import Path - -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -# Only content files go to semantic extraction. Code is already covered -# structurally by the AST pass; flattening every category here makes the -# extraction step re-read every source file (#1392). -all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] - -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) - -# Always (re)write the cache file: write hits, else DELETE any leftover from a -# prior run so Part C never merges a stale .graphify_cached.json (#1392). -if cached_nodes or cached_edges or cached_hyperedges: - Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) -else: - Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) -Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached)) -print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') -" -``` - -Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip to Part C directly. - -**Step B1 - Split into chunks** - -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. - -**Step B2 - Dispatch subagents** - -Dispatch ALL subagents in the same response so they run in parallel. - -Concrete example for 3 chunks: -``` -[Subagent 1: files 1-15] -[Subagent 2: files 16-30] -[Subagent 3: files 31-45] -``` -All three in one message. Not three separate messages. - -For each chunk, dispatch a subagent with this exact prompt (fill in FILE_LIST): - -``` -You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment. -Output ONLY valid JSON with no commentary: {"nodes": [...], "edges": [...], "hyperedges": [...], "input_tokens": 0, "output_tokens": 0} - -Extraction rules: -- EXTRACTED: relationship explicit in source (import, call, citation) -- INFERRED: reasonable inference (shared structure, implied dependency) -- AMBIGUOUS: uncertain — flag it, do not omit -- Code files: extract semantic edges AST cannot find (design patterns, protocol conformance). Do not re-extract imports. -- Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. When adding `calls` edges: source is caller, target is callee. -- Image files: use vision to understand what the image IS - do not just OCR. - UI screenshot: layout patterns, design decisions, key elements, purpose. - Chart: metric, trend/insight, data source. - Tweet/post: claim as node, author, concepts mentioned. - Diagram: components and connections. - Research figure: what it demonstrates, method, result. - Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS. -- DEEP_MODE (if set): be aggressive with INFERRED edges -- Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. -- Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. -- If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, - contributor onto every node from that file. -- confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: -- EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: pick exactly ONE value from this set — never 0.5: - 0.95 direct structural evidence (shared data structure, named cross-file reference). - 0.85 strong inference (clear functional alignment, no direct symbol link). - 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). - 0.65 weak inference (thematically related, no shape evidence). - 0.55 speculative but plausible (surface-level co-occurrence only). - Models follow discrete rubrics better than continuous ranges; the bimodal - distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the - range guidance is being collapsed to a binary. If no value above fits, mark - the edge AMBIGUOUS rather than picking 0.4 or below. -- AMBIGUOUS edges: 0.1-0.3 - -Schema: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} - -Files: -FILE_LIST -``` - -**Step B3 - Cache and merge** - -Wait for all subagents. For each result: -- Check that `graphify-out/.graphify_chunk_N.json` exists on disk — this is the success signal -- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache -- If the file is missing, the subagent was likely dispatched as read-only — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip. -- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort - -If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. - -After each subagent call completes, write its result to `graphify-out/.graphify_chunk_N.json`. **After each subagent call completes, read the real token counts from the subagent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then merge: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, glob -from pathlib import Path -from graphify.semantic_cleanup import load_validated_semantic_fragment, sanitize_semantic_fragment - -chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) -all_nodes, all_edges, all_hyperedges = [], [], [] -total_in, total_out = 0, 0 -for c in chunks: - d, errors = load_validated_semantic_fragment(Path(c)) - if errors: - print(f'Skipping invalid chunk {c}: ' + '; '.join(errors[:3])) - continue - d = sanitize_semantic_fragment(d) - all_nodes += d.get('nodes', []) - all_edges += d.get('edges', []) - all_hyperedges += d.get('hyperedges', []) - total_in += d.get('input_tokens', 0) - total_out += d.get('output_tokens', 0) -Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ - 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, - 'input_tokens': total_in, 'output_tokens': total_out, -}, indent=2)) -print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') -" -``` - -Save new results to cache: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.cache import save_semantic_cache -from pathlib import Path - -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) -print(f'Cached {saved} files') -" -``` - -Merge cached + new results into `graphify-out/.graphify_semantic.json`: -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment - -cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text()) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} - -all_nodes = cached['nodes'] + new.get('nodes', []) -all_edges = cached['edges'] + new.get('edges', []) -all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) -seen = set() -deduped = [] -for n in all_nodes: - if n['id'] not in seen: - seen.add(n['id']) - deduped.append(n) - -merged = { - 'nodes': deduped, - 'edges': all_edges, - 'hyperedges': all_hyperedges, - 'input_tokens': new.get('input_tokens', 0), - 'output_tokens': new.get('output_tokens', 0), -} -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) -print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') -" -``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Use bounded semantic chunks only for non-code content. #### Part C - Merge AST + semantic into final extraction -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from pathlib import Path -from graphify.semantic_cleanup import sanitize_semantic_fragment - -ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text()) -sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text()) - -# Merge: AST nodes first, semantic nodes deduplicated by id -seen = {n['id'] for n in ast['nodes']} -merged_nodes = list(ast['nodes']) -for n in sem['nodes']: - if n['id'] not in seen: - merged_nodes.append(n) - seen.add(n['id']) - -merged_edges = ast['edges'] + sem['edges'] -merged_hyperedges = sem.get('hyperedges', []) -merged = { - 'nodes': merged_nodes, - 'edges': merged_edges, - 'hyperedges': merged_hyperedges, - 'input_tokens': sem.get('input_tokens', 0), - 'output_tokens': sem.get('output_tokens', 0), -} -merged = sanitize_semantic_fragment(merged) -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) -total = len(merged_nodes) -edges = len(merged_edges) -print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') -" -``` +Transient DTOs are validated by the parent and committed with native state. ### Step 4 - Build graph, cluster, analyze, generate outputs -**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source->target), otherwise `False` (the default undirected `Graph`). Substitute it everywhere it appears, the same way you substitute `INPUT_PATH` - do not leave the literal `IS_DIRECTED` in the code. - -```bash -mkdir -p graphify-out -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from graphify.export import to_json -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -# Guard BEFORE any write: an empty extraction must not clobber a good graph.json / -# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392). -if G.number_of_nodes() == 0: - print('ERROR: Graph is empty - extraction produced no nodes.') - print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') - raise SystemExit(1) -communities = cluster(G) -cohesion = score_all(G, communities) -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} -# Placeholder questions - regenerated with real labels in Step 5 -questions = suggest_questions(G, communities, labels) - -# Persist the graph first and only write the report/analysis if it actually -# persisted - to_json refuses to shrink an existing graph.json (#479), and a -# report describing a graph we did not write would be a lie (#1392). -wrote = to_json(G, communities, 'graphify-out/graph.json') -if not wrote: - print('ERROR: refused to shrink graphify-out/graph.json (fewer nodes than the existing graph). Run a full rebuild to be safe.') - raise SystemExit(1) -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, - 'questions': questions, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') -" -``` - -If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. - -Replace INPUT_PATH with the actual path. +Run `graphify extract INPUT_PATH`. Atomic activation deletes inactive generations by default; pass `--retain-rollback` to retain exactly one previous generation. ### Step 5 - Label communities -Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). - -Then regenerate the report and save the labels for the visualizer: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.cluster import score_all -from graphify.analyze import god_nodes, surprising_connections, suggest_questions -from graphify.report import generate -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} - -# LABELS - replace these with the names you chose above -labels = LABELS_DICT - -# Regenerate questions with real community labels (labels affect question phrasing) -questions = suggest_questions(G, communities, labels) - -report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) -print('Report updated with community labels') -" -``` - -Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). -Replace INPUT_PATH with the actual path. +Run `graphify label . --missing-only`; labels remain inside Helix state. ### Step 6 - Generate Obsidian vault (opt-in) + HTML -**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. - -If `--obsidian` was given: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_obsidian, to_canvas -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) -print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') - -to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) -print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') -print() -print('Open graphify-out/obsidian/ as a vault in Obsidian.') -print(' Graph view - nodes colored by community (set automatically)') -print(' graph.canvas - structured layout with communities as groups') -print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') -" -``` - -Generate the HTML graph (always, unless `--no-viz`): - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_html -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -NODE_LIMIT = 5000 -if G.number_of_nodes() > NODE_LIMIT: - from collections import Counter - print(f'Graph has {G.number_of_nodes()} nodes (above {NODE_LIMIT} limit). Building aggregated community view...') - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - import networkx as nx_meta - meta = nx_meta.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=labels.get(cid, f'Community {cid}')) - edge_counts = Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, relation=f'{w} cross-community edges', confidence='AGGREGATED') - if meta.number_of_nodes() > 1: - meta_communities = {cid: [str(cid)] for cid in communities} - member_counts = {cid: len(members) for cid, members in communities.items()} - to_html(meta, meta_communities, 'graphify-out/graph.html', community_labels=labels or None, member_counts=member_counts) - print(f'graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)') - print('Tip: run with --obsidian for full node-level detail, or --wiki for an agent-crawlable wiki.') - else: - print('Single community — aggregated view not useful. Skipping graph.html.') -else: - to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) - print('graph.html written - open in any browser, no server needed') -" -``` +Run native `graphify export` commands. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -The wiki is an agent-crawlable export — `index.md` plus one article per community plus god-node articles. It is the recommended fallback for graphs too large to render as HTML, and it's the most useful output for an autonomous agent navigating the graph between sessions. - -Run this before Step 9 (cleanup) so `graphify-out/.graphify_labels.json` is still available. - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.wiki import to_wiki -from graphify.analyze import god_nodes -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -cohesion = {int(k): v for k, v in analysis['cohesion'].items()} -labels = {int(k): v for k, v in labels_raw.items()} -gods = god_nodes(G) - -n = to_wiki(G, communities, 'graphify-out/wiki', community_labels=labels or None, cohesion=cohesion, god_nodes_data=gods) -print(f'Wiki: {n} articles written to graphify-out/wiki/') -print(' graphify-out/wiki/index.md -> agent entry point') -" -``` +Run `graphify export wiki graphify-out/graph.helix`. ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_cypher -from pathlib import Path - -G = build_from_json(json.loads(Path('graphify-out/.graphify_extract.json').read_text()), directed=IS_DIRECTED) -to_cypher(G, 'graphify-out/cypher.txt') -print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') -" -``` - -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import push_to_neo4j -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) -print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') -" -``` - -Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. +Run `graphify export neo4j graphify-out/graph.helix`. ### Step 7b - SVG export (only if --svg flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_svg -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) -labels_raw = json.loads(Path('graphify-out/.graphify_labels.json').read_text()) if Path('graphify-out/.graphify_labels.json').exists() else {} - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} -labels = {int(k): v for k, v in labels_raw.items()} - -to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) -print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') -" -``` +Run `graphify export svg graphify-out/graph.helix`. ### Step 7c - GraphML export (only if --graphml flag) -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.build import build_from_json -from graphify.export import to_graphml -from pathlib import Path - -extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text()) - -G = build_from_json(extraction, directed=IS_DIRECTED) -communities = {int(k): v for k, v in analysis['communities'].items()} - -to_graphml(G, communities, 'graphify-out/graph.graphml') -print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') -" -``` +Run `graphify export graphml graphify-out/graph.helix`. ### Step 7d - MCP server (only if --mcp flag) -```bash -python3 -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`: -```json -{ - "mcpServers": { - "graphify": { - "command": "python3", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} -``` +Run `python3 -m graphify.serve graphify-out/graph.helix`. ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.benchmark import run_benchmark, print_benchmark -from pathlib import Path - -detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) -print_benchmark(result) -" -``` - -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. - ---- +Run `graphify benchmark --graph graphify-out/graph.helix`. ### Step 9 - Save manifest, update cost tracker, clean up, and report -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from datetime import datetime, timezone -from graphify.detect import save_manifest - -# Save manifest for --update -detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') - -# Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -input_tok = extract.get('input_tokens', 0) -output_tok = extract.get('output_tokens', 0) - -cost_path = Path('graphify-out/cost.json') -if cost_path.exists(): - cost = json.loads(cost_path.read_text()) -else: - cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} - -cost['runs'].append({ - 'date': datetime.now(timezone.utc).isoformat(), - 'input_tokens': input_tok, - 'output_tokens': output_tok, - 'files': detect.get('total_files', 0), -}) -cost['total_input_tokens'] += input_tok -cost['total_output_tokens'] += output_tok -cost_path.write_text(json.dumps(cost, indent=2)) - -print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') -print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') -" -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_incremental.json graphify-out/.graphify_transcripts.json graphify-out/.graphify_old.json; find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null -rm -f graphify-out/.needs_update 2>/dev/null || true -``` - -Tell the user (omit the obsidian line unless --obsidian was given; omit the wiki line unless --wiki was given): -``` -Graph complete. Outputs in PATH_TO_DIR/graphify-out/ - - graph.html - interactive graph, open in browser - GRAPH_REPORT.md - audit report - graph.json - raw graph data - obsidian/ - Obsidian vault (only if --obsidian was given) - wiki/ - agent-crawlable wiki, start at wiki/index.md (only if --wiki was given) -``` - -If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi - -Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. - -Then paste these sections from GRAPH_REPORT.md directly into the chat: -- God Nodes -- Surprising Connections -- Suggested Questions - -Do NOT paste the full report - just those three sections. Keep it concise. - -Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: - -> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" - -If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. - -The graph is the map. Your job after the pipeline is to be the guide. - ---- +Build metadata, hashes, caches, and learning state are durable native state. ## Interpreter guard for subcommands -Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first: - -```bash -if [ ! -f graphify-out/.graphify_python ]; then - GRAPHIFY_BIN=$(which graphify 2>/dev/null) - if [ -n "$GRAPHIFY_BIN" ]; then - PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') - case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac - else - PYTHON="python3" - fi - mkdir -p graphify-out - "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" -fi -``` - ---- +Prefer `graphify`; otherwise use `python3 -m graphify`. ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result)) -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4-8. - -If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A-3C pipeline as normal. - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.build import build_from_json -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load existing graph -existing_data = json.loads(Path('graphify-out/graph.json').read_text()) -G_existing = json_graph.node_link_graph(existing_data, edges='links') - -# Load new extraction -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extraction, directed=IS_DIRECTED) - -# Merge: new nodes/edges into existing graph -G_existing.update(G_new) -print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') -" -``` - -Then run Steps 4-8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text()) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` - -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` - ---- +Run `graphify update INPUT_PATH`. ## For --cluster-only -Skip Steps 1-3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.cluster import cluster, score_all -from graphify.analyze import god_nodes, surprising_connections -from graphify.report import generate -from graphify.export import to_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, - 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output': 0} - -communities = cluster(G) -cohesion = score_all(G, communities) -gods = god_nodes(G) -surprises = surprising_connections(G, communities) -labels = {cid: 'Community ' + str(cid) for cid in communities} - -report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') -Path('graphify-out/GRAPH_REPORT.md').write_text(report) -to_json(G, communities, 'graphify-out/graph.json') - -analysis = { - 'communities': {str(k): v for k, v in communities.items()}, - 'cohesion': {str(k): v for k, v in cohesion.items()}, - 'gods': gods, - 'surprises': surprises, -} -Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) -print(f'Re-clustered: {len(communities)} communities') -" -``` - -Then run Steps 5-9 as normal (label communities, generate viz, benchmark, clean up, report). - ---- +Run `graphify cluster-only INPUT_PATH`. ## For /graphify query -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. - -Load `graphify-out/graph.json`, then: - -1. Find the 1-3 nodes whose label best matches key terms in the question. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) > 3] - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` - -Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. - -After writing the answer, save it back into the graph so it improves future queries: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 -``` - ---- +Run `graphify query "QUESTION" [--dfs] [--budget N]`. ## For /graphify path -Find the shortest path between two named concepts in the graph. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" -``` - -Replace `NODE_A` and `NODE_B` with the actual concept names. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Run `graphify path "A" "B" --graph graphify-out/graph.helix`. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text()) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept. Then write a 3-5 sentence explanation using source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` - ---- +Run `graphify explain "NODE" --graph graphify-out/graph.helix`. ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" -``` - -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- Twitter/X -> fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv -> abstract + metadata saved as `.md` -- PDF -> downloaded as `.pdf` -- Images (.png/.jpg/.webp) -> downloaded, vision extraction runs on next build -- Any webpage -> converted to markdown via html2text - ---- +Run `graphify add URL`, then `graphify update .`. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - -```bash -python3 -m graphify.watch INPUT_PATH --debounce 3 -``` - -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - ---- +Run `graphify watch INPUT_PATH`. ## For git commit hook -Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. - -```bash -graphify hook install # install -graphify hook uninstall # remove -graphify hook status # check -``` - -After every `git commit`, the hook detects which code files changed, re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. - ---- +Run `graphify hook install`. ## For always-on context in Devin sessions -Run once per project to make graphify always-on in Devin sessions: - -```bash -graphify devin install --project -``` - -This writes a `## graphify` section to `.windsurf/rules/graphify.md` that instructs Devin to check the graph before answering codebase questions and rebuild it after code changes. - -```bash -graphify devin uninstall --project # remove -``` - ---- +Tell Devin to query `graphify-out/graph.helix` before broad source search. ## Honesty Rules -- Never invent an edge. If unsure, use AMBIGUOUS. -- Never skip the corpus check warning. -- Always show token cost in the report. -- Never hide cohesion scores behind symbols - show the raw number. +- Never invent an edge. +- Never reconstruct or migrate obsolete runtime data. +- Preserve confidence tags and source citations. diff --git a/tools/skillgen/fragments/extra/powershell-troubleshooting.md b/tools/skillgen/fragments/extra/powershell-troubleshooting.md index f7a35137e..368c8f522 100644 --- a/tools/skillgen/fragments/extra/powershell-troubleshooting.md +++ b/tools/skillgen/fragments/extra/powershell-troubleshooting.md @@ -1,12 +1,11 @@ ## Troubleshooting -### PowerShell 5.1: Vertical scrolling stops working +### Windows support + +Use CPython 3.10 or 3.12 on native Windows x86_64 and install Graphify normally with pip or uv. The exact public `helix-db-embedded` version must provide a `win_amd64` wheel; do not substitute WSL, a source build, a downloaded DLL, or a compatibility graph library. -If vertical scrolling breaks in PowerShell after running graphify, this is caused by ANSI escape sequences from the `graspologic` library. Graphify v0.3.10+ suppresses this output, but if you still see the issue: +### PowerShell 5.1: Vertical scrolling stops working -1. **Upgrade graphify**: `pip install --upgrade graphifyy` -2. **Use Windows Terminal** instead of the legacy PowerShell console — Windows Terminal handles ANSI codes correctly -3. **Reset your terminal**: close and reopen PowerShell -4. **Skip graspologic**: uninstall it (`pip uninstall graspologic`) and graphify will fall back to NetworkX's built-in Louvain algorithm, which produces no ANSI output +Use Windows Terminal or PowerShell 7 when possible. Graphify does not patch terminal modes or load a helper DLL. --- diff --git a/tools/skillgen/fragments/query-stub/default.md b/tools/skillgen/fragments/query-stub/default.md index 696796ec5..2f488724d 100644 --- a/tools/skillgen/fragments/query-stub/default.md +++ b/tools/skillgen/fragments/query-stub/default.md @@ -1,7 +1,7 @@ -When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it: +When `graphify-out/graph.helix` exists, expand the question against the graph's own vocabulary and answer from its active native generation: ```bash graphify query "" ``` -Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`. +Use `--dfs` for a trace and `--budget N` to cap output. There is no JSON or in-process compatibility fallback. If the CLI cannot open the Helix store, ask for a source rebuild. See `references/query.md` for query, path, explain, and feedback flows. diff --git a/tools/skillgen/fragments/references/host/hooks-agents-md.md b/tools/skillgen/fragments/references/host/hooks-agents-md.md index 26b9a8f74..c4cb5dd96 100644 --- a/tools/skillgen/fragments/references/host/hooks-agents-md.md +++ b/tools/skillgen/fragments/references/host/hooks-agents-md.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/fragments/references/query/default.md b/tools/skillgen/fragments/references/query/default.md index 56565eb78..082e28887 100644 --- a/tools/skillgen/fragments/references/query/default.md +++ b/tools/skillgen/fragments/references/query/default.md @@ -1,311 +1,43 @@ # graphify reference: query, path, explain -Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise. - -Two traversal modes - choose based on the question: - -| Mode | Flag | Best for | -|------|------|----------| -| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | -| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | - -First check the graph exists: -```bash -$(cat graphify-out/.graphify_python) -c " -from pathlib import Path -if not Path('graphify-out/graph.json').exists(): - print('ERROR: No graph found. Run /graphify first to build the graph.') - raise SystemExit(1) -" -``` -If it fails, stop and tell the user to run `/graphify ` first. +Use this reference only when an active `graphify-out/graph.helix` store exists. Graphify resolves labels, scores vocabulary, traverses, and renders evidence directly from the immutable native snapshot. ### Step 0 — Constrained query expansion (REQUIRED before traversal) -graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise. - -Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first: - -1. Extract the token vocabulary from node labels: -```bash -$(cat graphify-out/.graphify_python) -c " -import json, re -from pathlib import Path -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -vocab = set() -for n in data['nodes']: - for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE): - parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c] - for p in parts: - t = p.lower() - if 3 <= len(t) <= 30: - vocab.add(t) -Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8') -print(f'vocab: {len(vocab)} tokens') -" -``` - -2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints: - - You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens. - - If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory. - - If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search. - - Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab. - - Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present. - -3. Print the selection explicitly to the user before running the query, so the expansion is auditable: -``` -Query expanded to (from graph vocab, N tokens): [token1, token2, ...] -``` -If the list is empty, say so plainly and stop — do not proceed to traversal. +Pass the user's wording to the native CLI. It expands terms against labels and identifiers stored in the active generation; do not reproduce that scoring in the skill. ### Step 1 — Traversal -Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.) - -Prefer the CLI when it is installed: -```bash -graphify query "QUESTION" -# or: graphify query "QUESTION" --dfs --budget 3000 -``` - -If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: - -1. Find the 1-3 nodes whose label best matches the expanded tokens. -2. Run the appropriate traversal from each starting node. -3. Read the subgraph - node labels, edge relations, confidence tags, source locations. -4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. -5. If the graph lacks enough information, say so - do not hallucinate edges. - -```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -question = 'QUESTION' -mode = 'MODE' # 'bfs' or 'dfs' -terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392) - -# Find best-matching start nodes -scored = [] -for nid, ndata in G.nodes(data=True): - label = ndata.get('label', '').lower() - score = sum(1 for t in terms if t in label) - if score > 0: - scored.append((score, nid)) -scored.sort(reverse=True) -start_nodes = [nid for _, nid in scored[:3]] - -if not start_nodes: - print('No matching nodes found for query terms:', terms) - sys.exit(0) - -subgraph_nodes = set() -subgraph_edges = [] - -if mode == 'dfs': - # DFS: follow one path as deep as possible before backtracking. - # Depth-limited to 6 to avoid traversing the whole graph. - visited = set() - stack = [(n, 0) for n in reversed(start_nodes)] - while stack: - node, depth = stack.pop() - if node in visited or depth > 6: - continue - visited.add(node) - subgraph_nodes.add(node) - for neighbor in G.neighbors(node): - if neighbor not in visited: - stack.append((neighbor, depth + 1)) - subgraph_edges.append((node, neighbor)) -else: - # BFS: explore all neighbors layer by layer up to depth 3. - frontier = set(start_nodes) - subgraph_nodes = set(start_nodes) - for _ in range(3): - next_frontier = set() - for n in frontier: - for neighbor in G.neighbors(n): - if neighbor not in subgraph_nodes: - next_frontier.add(neighbor) - subgraph_edges.append((n, neighbor)) - subgraph_nodes.update(next_frontier) - frontier = next_frontier - -# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) -token_budget = BUDGET # default 2000 -char_budget = token_budget * 4 - -# Score each node by term overlap for ranked output -def relevance(nid): - label = G.nodes[nid].get('label', '').lower() - return sum(1 for t in terms if t in label) - -ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) - -lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] -for nid in ranked_nodes: - d = G.nodes[nid] - lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') -for u, v in subgraph_edges: - if u in subgraph_nodes and v in subgraph_nodes: - _raw = G[u][v]; d = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') - -output = '\n'.join(lines) -if len(output) > char_budget: - output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' -print(output) -" -``` +Two traversal modes are available: -Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains. +| Mode | Flag | Best for | +|------|------|----------| +| BFS | _(none)_ | Broad nearby context | +| DFS | `--dfs` | A focused dependency or call trace | -After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node: +Run: ```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +graphify query "QUESTION" +graphify query "QUESTION" --dfs --budget 3000 ``` -Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. - -**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting): - -- `useful` — the cited nodes answered the question well (they become *preferred sources*). -- `dead_end` — the question/path led nowhere; don't re-derive it next time. -- `corrected` — the saved answer was wrong; `--correction` records what was right. +Answer only from returned nodes and edges. Preserve confidence tags and cite `source_file`/`source_location` when present. If no match exists, say that the native graph lacks evidence rather than inventing a relationship. -At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing. - ---- +Record useful, dead-end, or corrected outcomes with `graphify save-result`; the learning state is committed inside Helix and can be summarized with `graphify reflect --if-stale`. ## For /graphify path -Find the shortest path between two named concepts in the graph. Prefer the CLI when installed: - -```bash -graphify path "NODE_A" "NODE_B" -``` - -If the CLI is unavailable, run it inline: - ```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -a_term = 'NODE_A' -b_term = 'NODE_B' - -def find_node(term): - term = term.lower() - scored = sorted( - [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True - ) - return scored[0][1] if scored and scored[0][0] > 0 else None - -src = find_node(a_term) -tgt = find_node(b_term) - -if not src or not tgt: - print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') - sys.exit(0) - -try: - path = nx.shortest_path(G, src, tgt) - print(f'Shortest path ({len(path)-1} hops):') - for i, nid in enumerate(path): - label = G.nodes[nid].get('label', nid) - if i < len(path) - 1: - _raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - print(f' {label} --{rel}--> [{conf}]') - else: - print(f' {label}') -except nx.NetworkXNoPath: - print(f'No path found between {a_term!r} and {b_term!r}') -except nx.NodeNotFound as e: - print(f'Node not found: {e}') -" +graphify path "NODE_A" "NODE_B" --graph graphify-out/graph.helix ``` -Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B -``` - ---- +Explain each directed relation and confidence tag in the returned native shortest path. If there is no path, report that directly. ## For /graphify explain -Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed: - ```bash -graphify explain "NODE_NAME" +graphify explain "NODE_NAME" --graph graphify-out/graph.helix ``` -If the CLI is unavailable, run it inline: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json, sys -import networkx as nx -from networkx.readwrite import json_graph -from pathlib import Path - -data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8')) -G = json_graph.node_link_graph(data, edges='links') - -term = 'NODE_NAME' -term_lower = term.lower() - -# Find best matching node -scored = sorted( - [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) - for n in G.nodes()], - reverse=True -) -if not scored or scored[0][0] == 0: - print(f'No node matching {term!r}') - sys.exit(0) - -nid = scored[0][1] -data_n = G.nodes[nid] -print(f'NODE: {data_n.get(\"label\", nid)}') -print(f' source: {data_n.get(\"source_file\",\"unknown\")}') -print(f' type: {data_n.get(\"file_type\",\"unknown\")}') -print(f' degree: {G.degree(nid)}') -print() -print('CONNECTIONS:') -for neighbor in G.neighbors(nid): - _raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw - nlabel = G.nodes[neighbor].get('label', neighbor) - rel = edge.get('relation', '') - conf = edge.get('confidence', '') - src_file = G.nodes[neighbor].get('source_file', '') - print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') -" -``` - -Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. - -After writing the explanation, save it back: - -```bash -$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME -``` +Summarize the node, its source, degree, learning annotation, and native connections. Do not fall back to reading or reconstructing an obsolete graph file. diff --git a/tools/skillgen/fragments/references/shared/add-watch.md b/tools/skillgen/fragments/references/shared/add-watch.md index 77844343e..a6878f641 100644 --- a/tools/skillgen/fragments/references/shared/add-watch.md +++ b/tools/skillgen/fragments/references/shared/add-watch.md @@ -1,56 +1,18 @@ # graphify reference: add a URL and watch a folder -Load this when the user ran `/graphify add ` or passed `--watch`. Neither is part of the default build. - ## For /graphify add -Fetch a URL and add it to the corpus, then update the graph. - ```bash -$(cat graphify-out/.graphify_python) -c " -import sys -from graphify.ingest import ingest -from pathlib import Path - -try: - out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') - print(f'Saved to {out}') -except ValueError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -except RuntimeError as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" +graphify add URL --dir raw +graphify update . ``` -Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. - -Supported URL types (auto-detected): -- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`) -- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author -- arXiv → abstract + metadata saved as `.md` -- PDF → downloaded as `.pdf` -- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text - ---- +Use `--author` and `--contributor` when supplied. The fetched source is ordinary corpus input; the update commits it to a new native generation. ## For --watch -Start a background watcher that monitors a folder and auto-updates the graph when files change. - ```bash -$(cat graphify-out/.graphify_python) -m graphify.watch INPUT_PATH --debounce 3 +graphify watch INPUT_PATH ``` -Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: - -- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. -- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). - -Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. - -Press Ctrl+C to stop. - -For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. +Watch mode keeps long-lived native readers, debounces file events, excludes concurrent writers, and atomically activates successful updates. An obsolete-format file is ignored with a rebuild warning. diff --git a/tools/skillgen/fragments/references/shared/exports.md b/tools/skillgen/fragments/references/shared/exports.md index 242ff868e..2d226ecb5 100644 --- a/tools/skillgen/fragments/references/shared/exports.md +++ b/tools/skillgen/fragments/references/shared/exports.md @@ -1,87 +1,48 @@ # graphify reference: extra exports and benchmark -Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag. +Every exporter opens an immutable native generation. No intermediate graph file is produced. ### Step 6b - Wiki (only if --wiki flag) -**Only run this step if `--wiki` was explicitly given in the original command.** - -Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available. - ```bash -graphify export wiki +graphify export wiki graphify-out/graph.helix --out graphify-out/wiki ``` ### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) -**If `--neo4j`** - generate a Cypher file for manual import: - ```bash -graphify export neo4j +graphify export neo4j graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: - -```bash -graphify export neo4j --push bolt://localhost:7687 --user neo4j --password PASSWORD -``` - -Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag) -**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact: - ```bash -graphify export falkordb +graphify export falkordb graphify-out/graph.helix --out graphify-out/cypher.txt ``` -**If `--falkordb-push `** - push directly to a running FalkorDB instance. Credentials are optional; ask the user only if the instance requires auth: - -```bash -graphify export falkordb --push falkordb://localhost:6379 -``` - -Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates. - ### Step 7b - SVG export (only if --svg flag) ```bash -graphify export svg +graphify export svg graphify-out/graph.helix --out graphify-out/graph.svg ``` ### Step 7c - GraphML export (only if --graphml flag) ```bash -graphify export graphml +graphify export graphml graphify-out/graph.helix --out graphify-out/graph.graphml ``` ### Step 7d - MCP server (only if --mcp flag) ```bash -$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json -``` - -This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. - -To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`: -```json -{ - "mcpServers": { - "graphify": { - "command": "", - "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] - } - } -} +python3 -m graphify.serve graphify-out/graph.helix +python3 -m graphify.serve graphify-out/graph.helix --transport http --port 8080 ``` ### Step 8 - Token reduction benchmark (only if total_words > 5000) -If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run: - ```bash -graphify benchmark +graphify benchmark --graph graphify-out/graph.helix ``` -Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. +Report generation, open, query, traversal, clustering, centrality, export, disk, RSS, and concurrent-reader measurements when running qualification benchmarks. diff --git a/tools/skillgen/fragments/references/shared/github-and-merge.md b/tools/skillgen/fragments/references/shared/github-and-merge.md index a41ea06e1..7684191af 100644 --- a/tools/skillgen/fragments/references/shared/github-and-merge.md +++ b/tools/skillgen/fragments/references/shared/github-and-merge.md @@ -1,46 +1,17 @@ -# graphify reference: GitHub clone and cross-repo merge - -Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph. +# graphify reference: GitHub clone and cross-repo aggregation ### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) -**Single repo:** -```bash -LOCAL_PATH=$(graphify clone [--branch ]) -# Use LOCAL_PATH as the target for all subsequent steps -``` - -**Multiple repos (cross-repo graph):** ```bash -# Clone each repo, run the full pipeline on each, then merge -graphify clone # → ~/.graphify/repos// -graphify clone # → ~/.graphify/repos// -# Run /graphify on each local path to produce their graph.json files -# Then merge: -graphify merge-graphs \ - ~/.graphify/repos///graphify-out/graph.json \ - ~/.graphify/repos///graphify-out/graph.json \ - --out graphify-out/cross-repo-graph.json +graphify clone https://github.com/OWNER/REPO --branch BRANCH --out LOCAL_PATH +graphify extract LOCAL_PATH ``` -Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. - -**Multiple local subfolders (monorepo or multi-service layout):** - -The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path: +For several repositories, build each project independently, then register each project directory or native store: ```bash -graphify extract ./core/ # → ./core/graphify-out/graph.json -graphify extract ./service/ # → ./service/graphify-out/graph.json -graphify extract ./platform/ # → ./platform/graphify-out/graph.json -# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set - -# Then merge at the project root: -graphify merge-graphs \ - ./core/graphify-out/graph.json \ - ./service/graphify-out/graph.json \ - ./platform/graphify-out/graph.json \ - --out graphify-out/graph.json +graphify global add repo-a +graphify global add repo-b/graphify-out/graph.helix ``` -Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate. +Global aggregation reads native snapshots. Do not combine storage files or invoke removed merge commands. diff --git a/tools/skillgen/fragments/references/shared/hooks.md b/tools/skillgen/fragments/references/shared/hooks.md index 438b8b16b..a908864c1 100644 --- a/tools/skillgen/fragments/references/shared/hooks.md +++ b/tools/skillgen/fragments/references/shared/hooks.md @@ -12,7 +12,7 @@ graphify hook uninstall # remove graphify hook status # check ``` -After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. +After every `git commit`, the hook detects changed code, stages a native update, and atomically activates `graphify-out/graph.helix` with its report. Doc/image changes require an explicit `graphify update .`. If a post-commit hook already exists, graphify appends to it rather than replacing it. diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index fa2612180..740e6b139 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -1,192 +1,25 @@ # graphify reference: incremental update and cluster-only -Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file. - ## For --update (incremental re-extraction) -Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. +Run: ```bash -$(cat graphify-out/.graphify_python) -c " -import sys, json -from graphify.detect import detect_incremental, save_manifest -from pathlib import Path - -result = detect_incremental(Path('INPUT_PATH')) -new_total = result.get('new_total', 0) -print(json.dumps(result, indent=2, ensure_ascii=False)) -Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\") -deleted = list(result.get('deleted_files', [])) -if new_total == 0 and not deleted: - print('No files changed since last run. Nothing to update.') - raise SystemExit(0) -if deleted: - print(f'{len(deleted)} deleted file(s) to prune.') -if new_total > 0: - print(f'{new_total} new/changed file(s) to re-extract.') -" +graphify update INPUT_PATH ``` -Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -Path('graphify-out/.graphify_detect.json').write_text(json.dumps({ - 'files': r.get('new_files', {}), - 'all_files': r.get('files', {}), - 'total_files': r.get('new_total', 0), - 'total_words': r.get('total_words', 0), - 'skipped_sensitive': r.get('skipped_sensitive', []), - 'needs_graph': True, -}, ensure_ascii=False), encoding=\"utf-8\") -" -``` - -If new files exist, first check whether all changed files are code files: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path - -result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {} -code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'} -new_files = result.get('new_files', {}) -all_changed = [f for files in new_files.values() for f in files] -code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) -print('code_only:', code_only) -" -``` - -If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. - -If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal. - - -If no new files exist (only deletions), create an empty extraction so the merge step can prune: - -```bash -if [ ! -f graphify-out/.graphify_extract.json ]; then - echo '[graphify update] Only deletions -- creating empty extraction for merge.' - $(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8') -" -fi -``` - - -Then: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from pathlib import Path -from graphify.build import build_merge -from graphify.detect import save_manifest - -# Load new extraction and incremental state -new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\")) -deleted = list(incremental.get('deleted_files', [])) -# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are -# handled by build_merge's replace-on-re-extract (#1344): every source_file in -# new_chunks is dropped from the base before merge, so old/stale nodes don't survive. -# Do NOT add `changed` here: with root= passed, prune_set relativizes to the same base -# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot -# now that replace — not the dedup pass — reconciles changed files). -prune = list(deleted) or None - -# Use build_merge() — reads graph.json directly without NetworkX round-trip -# so edge direction (calls, implements, imports) is always preserved (#801). -# Pass root= so prune_sources (absolute paths from detect_incremental) are -# relativized to match the graph's relative source_file values; without it -# nothing is pruned and stale nodes accumulate on every update (#1361). -# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else -# False. Without it a --directed --update silently rebuilds undirected and collapses -# reciprocal A<->B edges (#1392). -G = build_merge( - [new_extraction], - graph_path='graphify-out/graph.json', - prune_sources=prune, - root='INPUT_PATH', - directed=IS_DIRECTED, -) -print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges') - -# Write merged result back to .graphify_extract.json so Step 4 sees the full graph -merged_out = { - 'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)], - 'edges': [ - # Explicit source/target last so they win over any stale attrs in d. - {**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')}, - 'source': d.get('_src', u), 'target': d.get('_tgt', v)} - for u, v, d in G.edges(data=True) - ], - # G.graph["hyperedges"] holds hyperedges from both existing graph.json - # and new_extraction (build_merge combines them). Falling back to - # new_extraction only would silently drop prior-run hyperedges (#801). - 'hyperedges': list(G.graph.get('hyperedges', [])), - 'input_tokens': new_extraction.get('input_tokens', 0), - 'output_tokens': new_extraction.get('output_tokens', 0), -} -Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\") -print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') - -# Save manifest so next --update diffs against today's state, not the -# prior run's baseline (prevents ghost-node reports on subsequent updates). -# root= matches the build_merge call above so the manifest keys stay relative to -# the scan root — portable across clones/machines, so --update keeps matching -# cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') -print('[graphify update] Manifest saved.') -" -``` - -Then run Steps 4–8 on the merged graph as normal. - -After Step 4, show the graph diff: - -```bash -$(cat graphify-out/.graphify_python) -c " -import json -from graphify.analyze import graph_diff -from graphify.build import build_from_json -from networkx.readwrite import json_graph -import networkx as nx -from pathlib import Path - -# Load old graph (before update) from backup written before merge -old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None -new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) -G_new = build_from_json(new_extract, directed=IS_DIRECTED) - -if old_data: - G_old = json_graph.node_link_graph(old_data, edges='links') - diff = graph_diff(G_old, G_new) - print(diff['summary']) - if diff['new_nodes']: - print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) - if diff['new_edges']: - print('New edges:', len(diff['new_edges'])) -" -``` +The updater opens the active `graphify-out/graph.helix` generation, compares source hashes from native state, re-extracts changed files, removes deleted-source records, and stages a complete replacement generation. Extraction-cache changes, topology, communities, analysis, and hashes activate atomically. Concurrent readers keep their immutable snapshot; writers are excluded by the store lock. -Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json` -Clean up after: `rm -f graphify-out/.graphify_old.json` +If activation fails, the previous generation remains active. Existing obsolete-format files are ignored and never migrated or deleted. ---- +Use `--force` only for an intentional full source rebuild; it clears cached extraction state before staging the new generation. ## For --cluster-only -Skip Steps 1–3. Re-run clustering on the existing graph: +Run: ```bash -graphify cluster-only . +graphify cluster-only INPUT_PATH ``` -`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual. +This reclusters the active native topology, refreshes analysis, and commits the result as a new generation while retaining the prior generation for rollback. diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index 93e403f27..f3942525d 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -133,11 +133,8 @@ def _v8_baseline_ref(platform_key: str) -> str: "verbose": "references/shared/extraction-spec.md", "compact": "references/shared/extraction-spec-compact.md", } -# Single unified query reference + stub: superior vocab-expansion (Step 0) plus -# CLI traversal plus inline NetworkX fallback, shipped to every platform. The -# capabilities used to be split across cli.md / cli-inline.md so no platform got -# both — Claude had expansion but no fallback, the rest had the fallback but the -# weaker matcher (#1325). +# Single unified query reference + stub. Vocabulary expansion and traversal are +# owned by the native CLI so generated skills never duplicate runtime logic. _QUERY_REFERENCE = "references/query/default.md" _QUERY_STUB = "query-stub/default.md" # The hooks reference is host-flavored. Most hosts read CLAUDE.md and wire @@ -590,16 +587,15 @@ def _git_show(ref: str) -> str: def _v8_available() -> bool: - """Whether origin/v8 is fetchable in this checkout. + """Whether the pinned pre-split v8 commit exists in this checkout. - The git-show validators (audit-coverage, monolith-roundtrip, - always-on-roundtrip) read blobs from origin/v8. CI's default shallow checkout - does not fetch that ref, so the validators set fetch-depth: 0 to fetch it. + The heading audit reads blobs from a pinned commit. CI's default shallow + checkout does not fetch that history, so the validator job uses fetch-depth 0. This probe lets the CLI skip with a clear, actionable message (rather than crash with a cryptic git error) when the ref is genuinely unreachable. """ result = subprocess.run( - ["git", "rev-parse", "--verify", "--quiet", "origin/v8"], + ["git", "cat-file", "-e", f"{_V8_BASELINE_SHA}^{{commit}}"], cwd=REPO_ROOT, capture_output=True, text=True, @@ -769,13 +765,13 @@ def _is_directed_fix_line(line: str) -> bool: """Whether a line is part of the ``--directed`` propagation fix (#1392). The monolith runbooks built every graph undirected, so a ``--directed`` run - silently collapsed reciprocal A<->B edges. Every ``build_from_json(...)`` call + silently collapsed reciprocal A<->B edges. Every ``build_from_extraction(...)`` call now threads ``directed=IS_DIRECTED`` and a prose line tells the agent to substitute it like ``INPUT_PATH``. Both the old bare call (removed) and the new threaded call (added) match here, plus the substitution instruction. """ return ( - "build_from_json(" in line and "import" not in line + "build_from_extraction(" in line and "import" not in line ) or "directed=IS_DIRECTED" in line or ( "IS_DIRECTED" in line and "Substitute it everywhere" in line ) @@ -813,7 +809,7 @@ def _is_cache_unlink_fix_line(line: str) -> bool: def _is_zero_node_guard_fix_line(line: str) -> bool: """Whether a line is part of the zero-node / shrink-guard ordering fix (#1392). - Step 4 wrote GRAPH_REPORT.md, graph.json and the analysis sidecar *before* + Step 4 wrote graph outputs and the analysis sidecar *before* the zero-node guard, so an empty extraction clobbered a good graph; and the report was written even when ``to_json`` refused to shrink (#479). The guard now runs before any write and the report/analysis are gated on ``to_json``. @@ -827,11 +823,11 @@ def _is_zero_node_guard_fix_line(line: str) -> bool: or s == "raise SystemExit(1)" or "to_json(G, communities," in line or s == "if not wrote:" - or "refused to shrink graphify-out/graph.json" in line + or "refused to shrink the active graph" in line or "Guard BEFORE any write" in line or "GRAPH_REPORT.md / analysis sidecar" in line or "Persist the graph first" in line - or "to_json refuses to shrink an existing graph.json" in line + or "persistence refuses to shrink an existing graph" in line or "report describing a graph we did not write" in line ) @@ -945,45 +941,32 @@ def _is_sanctioned_monolith_diff(line: str) -> bool: def monolith_roundtrip(platform: Platform) -> list[str]: - """Assert a monolith renders diff-clean vs its v8 blob modulo allowed changes. - - The monolith bodies are hand-maintained single files frozen against a pinned - pristine v8 blob (``roundtrip_ref``); this is the guard that stops an - arbitrary edit (even a blessed one) from drifting them. Sanctioned changes are - enumerated as predicates in ``_SANCTIONED_MONOLITH_DIFFS``: the file_type enum - unification, the unified frontmatter description, the chunk-cleanup rewrite - (#1172), the four #1392 runbook fixes (directed propagation, content-only - semantic scope, stale-cache unlink, and the zero-node/shrink-guard ordering), - and semantic-cache source scoping (#1757). - - The comparison is a multiset diff, not a positional zip: a line whose text is - unchanged but merely *moved* (the report-write line shifted below ``to_json`` - in the ordering fix) cancels out and is not flagged. Only lines whose content - is genuinely added or removed are checked, and each must be sanctioned. + """Validate the Helix-only contract of a generated single-file skill. + + The old byte comparison against the legacy v8 runbook would preserve removed + storage and compatibility instructions. Monoliths now have an explicit native + contract instead: the sole runtime store is Helix, required public commands + remain documented, and removed runtime names cannot return. """ if platform.bucket != "monolith": return [] - if platform.roundtrip_ref is None: - return [f"[{platform.key}] monolith is missing roundtrip_ref"] - - rendered_lines = render(platform)[0].content.splitlines() - # Strip trigger lines from the original — they are non-spec and their removal - # (#1180) is a permitted diff. - original_lines = [ - l for l in _normalise(_git_show(platform.roundtrip_ref)).splitlines() - if not _is_trigger_line(l) - ] - - added = Counter(rendered_lines) - Counter(original_lines) - removed = Counter(original_lines) - Counter(rendered_lines) - + body = render(platform)[0].content problems: list[str] = [] - for line in list(added.elements()) + list(removed.elements()): - if _is_sanctioned_monolith_diff(line): - continue - problems.append( - f"[{platform.key}] unsanctioned monolith change vs pristine v8: {line!r}" - ) + required = ( + "graphify-out/graph.helix", + "graphify extract", + "graphify update", + "graphify query", + "graphify path", + "graphify explain", + ) + for marker in required: + if marker not in body: + problems.append(f"[{platform.key}] native monolith is missing {marker!r}") + folded = body.casefold() + for removed in ("networkx", "graspologic", "graph.json", "to_json("): + if removed in folded: + problems.append(f"[{platform.key}] native monolith retains removed term {removed!r}") return problems @@ -1013,36 +996,18 @@ def _always_on_constants(ref: str) -> dict[str, str]: def always_on_roundtrip() -> list[str]: - """Assert each always_on/*.md reproduces its former constant byte for byte. - - The six always-on instruction blocks were extracted from triple-quoted - constants in __main__.py into packaged markdown. This validator renders each - block and compares it, byte for byte, against the constant's value in the - pre-extraction source (ALWAYS_ON_BASELINE_REF). A mismatch means the - extraction is not faithful and the install-string / issue-#580 contract would - break. - """ - baseline = _always_on_constants(ALWAYS_ON_BASELINE_REF) + """Validate packaged always-on instructions against the Helix-only contract.""" problems: list[str] = [] rendered = {a.path: a.content for a in render_always_on()} for basename, const_name in sorted(ALWAYS_ON_BLOCKS.items()): path = f"graphify/always_on/{basename}.md" - if const_name not in baseline: - problems.append(f"could not find constant {const_name} in {ALWAYS_ON_BASELINE_REF}") - continue - expected = baseline[const_name] - for old, new in ALWAYS_ON_SANCTIONED_EDITS.get(const_name, ()): - if old not in expected: - problems.append( - f"sanctioned edit for {const_name} no longer applies: " - f"old text not found in {ALWAYS_ON_BASELINE_REF}" - ) - expected = expected.replace(old, new) - if rendered[path] != expected: - problems.append( - f"always_on/{basename}.md does not reproduce {const_name} byte for byte " - f"(rendered {len(rendered[path])} chars vs baseline {len(expected)} chars)" - ) + body = rendered[path] + if "graphify-out/graph.helix" not in body: + problems.append(f"always_on/{basename}.md does not name the native store") + folded = body.casefold() + for removed in ("networkx", "graspologic", "graph.json"): + if removed in folded: + problems.append(f"always_on/{basename}.md retains removed term {removed!r}") return problems @@ -1055,8 +1020,8 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: p.add_argument("--check", action="store_true", help="byte-diff render vs committed + expected/, exit 1 on drift") p.add_argument("--audit-coverage", action="store_true", help="per host: assert every heading of that host's own v8 body single-homes in its render") p.add_argument("--schema-singleton", action="store_true", help="assert the file_type enum is byte-identical everywhere") - p.add_argument("--monolith-roundtrip", action="store_true", help="assert each monolith == v8 modulo the enum unification") - p.add_argument("--always-on-roundtrip", action="store_true", help="assert each always_on/*.md reproduces its former __main__.py constant byte for byte") + p.add_argument("--monolith-roundtrip", action="store_true", help="validate each monolith's Helix-only runtime contract") + p.add_argument("--always-on-roundtrip", action="store_true", help="validate packaged always-on Helix instructions") p.add_argument("--bless", action="store_true", help="rewrite expected/ from the current render") return p.parse_args(argv) @@ -1065,15 +1030,12 @@ def main(argv: list[str] | None = None) -> int: args = _parse_args(argv if argv is not None else sys.argv[1:]) platforms = load_platforms() - # The git-show validators read origin/v8. On a shallow checkout that ref is - # absent; skip with a clear, actionable message instead of crashing. CI fixes - # this for real by setting fetch-depth: 0 so the validators actually run. - _GIT_SHOW_VALIDATORS = (args.audit_coverage, args.monolith_roundtrip, args.always_on_roundtrip) - if any(_GIT_SHOW_VALIDATORS) and not _v8_available(): + # The heading audit reads a pinned historical commit. A shallow checkout may + # omit it; CI fetches full history so this guard executes for real. + if args.audit_coverage and not _v8_available(): print( - "SKIPPED: origin/v8 is not fetchable in this checkout, so the git-show " - "validators cannot run. On CI, set fetch-depth: 0 on this job (actions/" - "checkout) so origin/v8 is fetched and the validators run for real.", + f"SKIPPED: pinned v8 baseline {_V8_BASELINE_SHA} is unavailable. " + "On CI, set fetch-depth: 0 so the heading audit can run.", file=sys.stderr, ) return 0 @@ -1115,7 +1077,7 @@ def main(argv: list[str] | None = None) -> int: for m in all_problems: print(f" {m}", file=sys.stderr) return 1 - print("monolith-roundtrip OK: each monolith matches v8 modulo the enum unification.") + print("monolith-roundtrip OK: each monolith satisfies the Helix-only contract.") return 0 if args.always_on_roundtrip: @@ -1125,7 +1087,7 @@ def main(argv: list[str] | None = None) -> int: for m in problems: print(f" {m}", file=sys.stderr) return 1 - print("always-on-roundtrip OK: each always_on/*.md reproduces its former constant byte for byte.") + print("always-on-roundtrip OK: each always_on/*.md satisfies the Helix-only contract.") return 0 artifacts = render_all(platforms, only=args.platform) diff --git a/uv.lock b/uv.lock index 088ebbbdc..4c0c4c1c7 100644 --- a/uv.lock +++ b/uv.lock @@ -65,15 +65,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] -[[package]] -name = "anytree" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bc/a8/eb55fab589c56f9b6be2b3fd6997aa04bb6f3da93b01154ce6fc8e799db2/anytree-2.13.0.tar.gz", hash = "sha256:c9d3aa6825fdd06af7ebb05b4ef291d2db63e62bb1f9b7d9b71354be9d362714", size = 48389, upload-time = "2025-04-08T21:06:30.662Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/98/f6aa7fe0783e42be3093d8ef1b0ecdc22c34c0d69640dfb37f56925cb141/anytree-2.13.0-py3-none-any.whl", hash = "sha256:4cbcf10df36b1f1cba131b7e487ff3edafc9d6e932a3c70071b5b768bab901ff", size = 45077, upload-time = "2025-04-08T21:06:29.494Z" }, -] - [[package]] name = "async-timeout" version = "5.0.1" @@ -92,18 +83,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] -[[package]] -name = "autograd" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/67/1c/3c24ec03c8ba4decc742b1df5a10c52f98c84ca8797757f313e7bdcdf276/autograd-1.8.0.tar.gz", hash = "sha256:107374ded5b09fc8643ac925348c0369e7b0e73bbed9565ffd61b8fd04425683", size = 2562146, upload-time = "2025-05-05T12:49:02.502Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/ea/e16f0c423f7d83cf8b79cae9452040fb7b2e020c7439a167ee7c317de448/autograd-1.8.0-py3-none-any.whl", hash = "sha256:4ab9084294f814cf56c280adbe19612546a35574d67c574b04933c7d2ecb7d78", size = 51478, upload-time = "2025-05-05T12:49:00.585Z" }, -] - [[package]] name = "av" version = "17.0.1" @@ -150,15 +129,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/a4/a26d5b25671d27e03afb5401a0be5899d94ff8fab6a698b1ac5be3ec29ef/bandit-1.9.4-py3-none-any.whl", hash = "sha256:f89ffa663767f5a0585ea075f01020207e966a9c0f2b9ef56a57c7963a3f6f8e", size = 134741, upload-time = "2026-02-25T06:44:13.694Z" }, ] -[[package]] -name = "beartype" -version = "0.18.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/15/4e623478a9628ad4cee2391f19aba0b16c1dd6fedcb2a399f0928097b597/beartype-0.18.5.tar.gz", hash = "sha256:264ddc2f1da9ec94ff639141fbe33d22e12a9f75aa863b83b7046ffff1381927", size = 1193506, upload-time = "2024-04-21T07:25:58.64Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/43/7a1259741bd989723272ac7d381a43be932422abcff09a1d9f7ba212cb74/beartype-0.18.5-py3-none-any.whl", hash = "sha256:5301a14f2a9a5540fe47ec6d34d758e9cd8331d36c4760fc7a5499ab86310089", size = 917762, upload-time = "2024-04-21T07:25:55.758Z" }, -] - [[package]] name = "beautifulsoup4" version = "4.14.3" @@ -477,7 +447,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } wheels = [ @@ -555,7 +525,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } @@ -813,10 +783,10 @@ name = "ctranslate2" version = "4.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, - { name = "pyyaml", marker = "python_full_version >= '3.11'" }, - { name = "setuptools", marker = "python_full_version >= '3.11'" }, + { name = "pyyaml" }, + { name = "setuptools" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/cb/e0/b69c40c3d739b213a78d327071240590792071b4f890e34088b03b95bb1e/ctranslate2-4.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9017a355dd7c6d29dc3bca6e9fc74827306c61b702c66bb1f6b939655e7de3fa", size = 1255773, upload-time = "2026-02-04T06:11:04.769Z" }, @@ -926,7 +896,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -951,12 +921,12 @@ name = "faster-whisper" version = "1.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "av", marker = "python_full_version >= '3.11'" }, - { name = "ctranslate2", marker = "python_full_version >= '3.11'" }, - { name = "huggingface-hub", marker = "python_full_version >= '3.11'" }, - { name = "onnxruntime", marker = "python_full_version >= '3.11'" }, - { name = "tokenizers", marker = "python_full_version >= '3.11'" }, - { name = "tqdm", marker = "python_full_version >= '3.11'" }, + { name = "av" }, + { name = "ctranslate2" }, + { name = "huggingface-hub" }, + { name = "onnxruntime" }, + { name = "tokenizers" }, + { name = "tqdm" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/05/99/49ee85903dee060d9f08297b4a342e5e0bcfca2f027a07b4ee0a38ab13f9/faster_whisper-1.2.1-py3-none-any.whl", hash = "sha256:79a66ad50688c0b794dd501dc340a736992a6342f7f95e5811be60b5224a26a7", size = 1118909, upload-time = "2025-10-31T11:35:47.794Z" }, @@ -1045,56 +1015,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] -[[package]] -name = "future" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/b2/4140c69c6a66432916b26158687e821ba631a4c9273c474343badf84d3ba/future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05", size = 1228490, upload-time = "2024-02-21T11:52:38.461Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326, upload-time = "2024-02-21T11:52:35.956Z" }, -] - -[[package]] -name = "gensim" -version = "4.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "smart-open", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1a/80/fe9d2e1ace968041814dbcfce4e8499a643a36c41267fa4b6c4f54cce420/gensim-4.4.0.tar.gz", hash = "sha256:a3f5b626da5518e79a479140361c663089fe7998df8ba52d56e1ded71ac5bdf5", size = 23260095, upload-time = "2025-10-18T02:06:45.962Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/88/1e7c7abf79cf88faca3d713fbb7068f58c9f44c77a3e72031cb3e40e43c3/gensim-4.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e29a2109819fdf5ff59bef670c8c22c1690d52239fe172b43e408908871de5f6", size = 24455330, upload-time = "2025-10-18T01:47:12.563Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2f/46a661db005730de7455090cb980b70147f04a3d162b49171582987d634e/gensim-4.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5c4d8f2a5e69bc246931dfd8e03d0ce3f3bcf82adbbdbcf20dfc35c43b8e1035", size = 24444343, upload-time = "2025-10-18T01:47:57.596Z" }, - { url = "https://files.pythonhosted.org/packages/a3/d8/ea8f98e198d8682c0d82cba04303d26f646ef2592a558739d812bfe02a3f/gensim-4.4.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0977e5e5df03f829f322662e37ac973b93272c526f1432f865d214c0b573f98", size = 27591522, upload-time = "2025-10-18T01:48:48.543Z" }, - { url = "https://files.pythonhosted.org/packages/7a/6e/9b835483f776ad0ab6fd1197441000c4005b0a3219d456b25296966f0107/gensim-4.4.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d56613fcb77d4068c1be845843508dcd9d384ede34700a61bbeac32b947d1fc3", size = 27631604, upload-time = "2025-10-18T01:49:39.557Z" }, - { url = "https://files.pythonhosted.org/packages/53/fe/e483909cfbfa8cc4bfd30aa9fb5170c04316cc22f23c9906529f08fb9095/gensim-4.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:724b93c9b6e92cd15837048c71b7fdd38059276c85dd1f9c0375576f0aea153f", size = 24395966, upload-time = "2025-10-18T01:50:24.398Z" }, - { url = "https://files.pythonhosted.org/packages/52/7b/81b6c74b32700ee63f6720a60ca0c89ab59b12933257b47572c8af017658/gensim-4.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7590e7313848ca8f3ff064898bcd6ecf6ec71c752cf4d3ec83f7ac992bc7c088", size = 24463159, upload-time = "2025-10-18T01:51:09.7Z" }, - { url = "https://files.pythonhosted.org/packages/38/7c/18d40f341276a7461962512ca1fb716d5982db57615dfa272f651ecb96d7/gensim-4.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:05a027238b5eb544a17afe73ec227d6a7e0c6b4e2108b1131c0b8f291a0e0e2e", size = 24453170, upload-time = "2025-10-18T01:51:58.42Z" }, - { url = "https://files.pythonhosted.org/packages/68/88/6bd6919d31bdd473472ce1c18c24fcab5869b8b15166a424d11ce33a5eab/gensim-4.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e110e2d3533f5b35239850a96cb2016a586ecd85671d655079b3048332b7169", size = 27760793, upload-time = "2025-10-18T01:52:47.866Z" }, - { url = "https://files.pythonhosted.org/packages/d9/fa/85531b39c1beb5a4203929ba83d94d886cec40d0fb0bef8ca05fd1cc7a38/gensim-4.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91a7fa5e814e7b1bad4b2dffa8d62c1e55410d5cbdf930714c1997ffb4404db8", size = 27809988, upload-time = "2025-10-18T01:53:36.978Z" }, - { url = "https://files.pythonhosted.org/packages/10/c3/7e22d6f7d88c4ea6a3a84481f00538252659d285713c3b7e2e1537b0e7e1/gensim-4.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5e2c1d584d1c7d16b2a0fe7d2f6f59a451422df7b5edb7e3ca46c8e462782127", size = 24396172, upload-time = "2025-10-18T01:54:25.711Z" }, - { url = "https://files.pythonhosted.org/packages/4f/65/d5285865ca54b93d41ccd8683c2d79952434957c76b411283c7a6c66ca69/gensim-4.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0845b2fa039dbea5667fb278b5414e70f6d48fd208ef51f33e84a78444288d8d", size = 24467245, upload-time = "2025-10-18T01:55:09.924Z" }, - { url = "https://files.pythonhosted.org/packages/32/59/f0ea443cbfb3b06e1d2e060217bb91f954845f6df38cbc9c5468b6c9c638/gensim-4.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1853fc5be730f692c444a826041fef9a2fc8d74c73bb59748904b2e3221daa86", size = 24455775, upload-time = "2025-10-18T01:55:52.866Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b8/9b0ba15756e41ccfdd852f9c65cd2b552f240c201dc3237ad8c178642e80/gensim-4.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23a2a4260f01c8f71bae5dd0e8a01bb247a2c789480c033e0eaba100b0ad4239", size = 27771345, upload-time = "2025-10-18T01:56:41.448Z" }, - { url = "https://files.pythonhosted.org/packages/97/2c/c29701826c963b04a43d5d7b87573a74040387ab9219e65b10f377d22b5b/gensim-4.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b73ff30af6ddd0d2ddf9473b1eb44603cd79ec14c87d93b75291802b991916c", size = 27864118, upload-time = "2025-10-18T01:57:32.428Z" }, - { url = "https://files.pythonhosted.org/packages/fd/f2/9ec6863143888bf390cdc5261f6d9e71d79bc95d98fb815679dba478d5f6/gensim-4.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b3a3f9bc8d4178b01d114e1c58c5ab2333f131c7415fb3d8ec8f1ecfe4c5b544", size = 24400277, upload-time = "2025-10-18T01:58:17.629Z" }, - { url = "https://files.pythonhosted.org/packages/80/6c/4e522973e07ca491d33cc7829996b9e8c8663a16b3f87f580cbdc2732d97/gensim-4.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8961b7a2bb5190b46bc6cd26c29d5bfea22f99123ed5f506ebd0aaf65996758", size = 24460186, upload-time = "2025-10-18T01:59:01.904Z" }, - { url = "https://files.pythonhosted.org/packages/cc/6a/593107ee98331128ed20e5d074865587558a0766659be787a40550ab66df/gensim-4.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:59d0d29099a76dd97d4563e002f3488a43e51f99d46387025da38007ebfeeff9", size = 24448880, upload-time = "2025-10-18T01:59:46.796Z" }, - { url = "https://files.pythonhosted.org/packages/d9/ef/1675e1a3a04f7d0293a21082f57f4a6a8bf0a9e387da58b71db648b663de/gensim-4.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3bec3e6a1ecaa6439b21a3e42ceb0ca67ffabc114b646f89b1aab5fe69a39ffc", size = 27736031, upload-time = "2025-10-18T02:00:36.791Z" }, - { url = "https://files.pythonhosted.org/packages/b3/b9/ee43ef9c391857232603a9ee281e9c5953f7922d70c98c2296a037d1c0b7/gensim-4.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9033b18920b7774e68eafacdbd87252ffa29382ec465ddb88bd036e00fc86365", size = 27826360, upload-time = "2025-10-18T02:01:26.166Z" }, - { url = "https://files.pythonhosted.org/packages/82/f3/4f8f4d478ce69af812c6002b513c5ad3242976923d172dbe5814903be22f/gensim-4.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:6ecb7aed37fb92d24e15a6adbabe693074003263db0fd9ce97c9f4234a9edc1b", size = 24396932, upload-time = "2025-10-18T02:02:11.568Z" }, -] - [[package]] name = "graphifyy" -version = "0.9.6" +version = "0.9.20" source = { editable = "." } dependencies = [ - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "defusedxml" }, + { name = "helix-db" }, + { name = "helix-db-embedded" }, { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "rapidfuzz" }, @@ -1132,7 +1060,6 @@ all = [ { name = "boto3" }, { name = "falkordb" }, { name = "faster-whisper", marker = "python_full_version >= '3.11'" }, - { name = "graspologic", marker = "python_full_version < '3.13'" }, { name = "jieba" }, { name = "markdownify" }, { name = "matplotlib" }, @@ -1178,9 +1105,6 @@ kimi = [ { name = "openai" }, { name = "tiktoken" }, ] -leiden = [ - { name = "graspologic", marker = "python_full_version < '3.13'" }, -] mcp = [ { name = "mcp" }, { name = "starlette" }, @@ -1252,23 +1176,23 @@ requires-dist = [ { name = "anthropic", marker = "extra == 'anthropic'" }, { name = "boto3", marker = "extra == 'all'" }, { name = "boto3", marker = "extra == 'bedrock'" }, + { name = "defusedxml", specifier = ">=0.7.1" }, { name = "falkordb", marker = "extra == 'all'" }, { name = "falkordb", marker = "extra == 'falkordb'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'all'" }, { name = "faster-whisper", marker = "python_full_version >= '3.11' and extra == 'video'" }, - { name = "graspologic", marker = "python_full_version < '3.13' and extra == 'all'" }, - { name = "graspologic", marker = "python_full_version < '3.13' and extra == 'leiden'" }, + { name = "helix-db", specifier = "==0.2.0b3" }, + { name = "helix-db-embedded", specifier = "==0.2.0b3" }, { name = "jieba", marker = "extra == 'all'" }, { name = "jieba", marker = "extra == 'chinese'" }, { name = "markdownify", marker = "extra == 'all'" }, { name = "markdownify", marker = "extra == 'pdf'" }, { name = "matplotlib", marker = "extra == 'all'" }, { name = "matplotlib", marker = "extra == 'svg'" }, - { name = "mcp", marker = "extra == 'all'" }, - { name = "mcp", marker = "extra == 'mcp'" }, + { name = "mcp", marker = "extra == 'all'", specifier = ">=1.28.1" }, + { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.28.1" }, { name = "neo4j", marker = "extra == 'all'" }, { name = "neo4j", marker = "extra == 'neo4j'" }, - { name = "networkx", specifier = ">=3.4" }, { name = "numpy", specifier = ">=1.21" }, { name = "numpy", marker = "python_full_version >= '3.13' and extra == 'all'", specifier = ">=2.0" }, { name = "numpy", marker = "python_full_version >= '3.13' and extra == 'svg'", specifier = ">=2.0" }, @@ -1331,7 +1255,7 @@ requires-dist = [ { name = "yt-dlp", marker = "extra == 'all'", specifier = ">=2026.6.9" }, { name = "yt-dlp", marker = "extra == 'video'", specifier = ">=2026.6.9" }, ] -provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "leiden", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "all"] +provides-extras = ["mcp", "neo4j", "falkordb", "pdf", "watch", "svg", "office", "google", "postgres", "video", "kimi", "ollama", "bedrock", "anthropic", "gemini", "openai", "chinese", "sql", "pascal", "dm", "terraform", "all"] [package.metadata.requires-dev] dev = [ @@ -1346,62 +1270,38 @@ dev = [ { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-cov", specifier = ">=7.1.0" }, { name = "ruff", specifier = ">=0.15.13" }, - { name = "setuptools", specifier = ">=82.0.1" }, + { name = "setuptools", specifier = ">=83.0.0" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" }, { name = "tree-sitter-hcl", specifier = ">=1.2.0" }, { name = "wheel", specifier = ">=0.47.0" }, ] [[package]] -name = "graspologic" -version = "3.4.4" +name = "h11" +version = "0.16.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anytree", marker = "python_full_version < '3.13'" }, - { name = "beartype", marker = "python_full_version < '3.13'" }, - { name = "future", marker = "python_full_version < '3.13'" }, - { name = "gensim", marker = "python_full_version < '3.13'" }, - { name = "graspologic-native", marker = "python_full_version < '3.13'" }, - { name = "hyppo", marker = "python_full_version < '3.13'" }, - { name = "joblib", marker = "python_full_version < '3.13'" }, - { name = "matplotlib", marker = "python_full_version < '3.13'" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "pot", marker = "python_full_version < '3.13'" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "seaborn", marker = "python_full_version < '3.13'" }, - { name = "statsmodels", marker = "python_full_version < '3.13'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, - { name = "umap-learn", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/bb/0fe2ef85ea775e7b8416b2cf90097aa4b5e0c9c2271d7fe6789bab27d0ca/graspologic-3.4.4.tar.gz", hash = "sha256:79878caf367da3e89046a4ec94291c5b1a5da569f19fdd879d8b45c3563d7110", size = 5134258, upload-time = "2025-09-08T21:44:01.969Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/b0/e26eb8fc25f3093ad168fba4101bdcf43258b91672546d20a2b64283845c/graspologic-3.4.4-py3-none-any.whl", hash = "sha256:4ea5cd50f10eaff3fa90f18a8f66b1f5f42c724ac6aeb95e9f081632fc8d2d00", size = 5200993, upload-time = "2025-09-08T21:43:59.843Z" }, + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] [[package]] -name = "graspologic-native" -version = "1.2.5" +name = "helix-db" +version = "0.2.0b3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/2d/62b30d89533643ccf4778a18eb023f291b8877b5d85de3342f07b2d363a7/graspologic_native-1.2.5.tar.gz", hash = "sha256:27ea7e01fa44466c0b4cdd678d4561e5d3dc0cb400015683b7ae1386031257a0", size = 2512729, upload-time = "2025-04-02T19:34:22.961Z" } +sdist = { url = "https://files.pythonhosted.org/packages/31/e5/746b5c6b078fb5385e00df7ad375e6047e898e5660154336afa59008d872/helix_db-0.2.0b3.tar.gz", hash = "sha256:3c4038913cb7ab91db803efa54d4d9ad14b4cd895367423cb9f771946ac6b477", size = 42482, upload-time = "2026-07-19T11:44:26.142Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/86/10748f4c474b0c8f6060dd379bb0c4da5d42779244bb13a58656ffb44a03/graspologic_native-1.2.5-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bf05f2e162ae2a2a8d6e8cfccbe3586d1faa0b808159ff950478348df557c61e", size = 648437, upload-time = "2025-04-02T19:34:16.29Z" }, - { url = "https://files.pythonhosted.org/packages/42/cc/b75ea35755340bedda29727e5388390c639ea533f55b9249f5ac3003f656/graspologic_native-1.2.5-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a7fff06ed49c3875cf351bb09a92ae7cbc169ce92dcc4c3439e28e801f822ae", size = 352044, upload-time = "2025-04-02T19:34:18.153Z" }, - { url = "https://files.pythonhosted.org/packages/8e/55/15e6e4f18bf249b529ac4cd1522b03f5c9ef9284a2f7bfaa1fd1f96464fe/graspologic_native-1.2.5-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e7e993e7d70fe0d860773fc62812fbb8cb4ef2d11d8661a1f06f8772593915", size = 364644, upload-time = "2025-04-02T19:34:19.486Z" }, - { url = "https://files.pythonhosted.org/packages/3b/51/21097af79f3d68626539ab829bdbf6cc42933f020e161972927d916e394c/graspologic_native-1.2.5-cp38-abi3-win_amd64.whl", hash = "sha256:c3ef2172d774083d7e2c8e77daccd218571ddeebeb2c1703cebb1a2cc4c56e07", size = 210438, upload-time = "2025-04-02T19:34:21.139Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d4/85118889f6946442b38867241f6e372914c16989cf98aef6e35b59304a13/helix_db-0.2.0b3-py3-none-any.whl", hash = "sha256:6d7f22e2d29949e740b5a245c1f082d140d3116f596dd3d16cccf8050fcb7003", size = 36430, upload-time = "2026-07-19T11:44:25.169Z" }, ] [[package]] -name = "h11" -version = "0.16.0" +name = "helix-db-embedded" +version = "0.2.0b3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3e/6df319091bf2681c08dbb25f74dddc60fc0fe767d73c713fc24c86c09ab4/helix_db_embedded-0.2.0b3-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5270ff1967b712e72ab86086edf0ba53d29a4a7ca1e72d043fb752aba0c019e6", size = 42290706, upload-time = "2026-07-19T11:44:50.598Z" }, + { url = "https://files.pythonhosted.org/packages/fe/12/0fa8ba4ac77b9997d04d2c662d8536579658f0148dcd2fd37be8e5d6476e/helix_db_embedded-0.2.0b3-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:84667a0eb081e2dd2770209553396079ed7bfa564de95e50b72be360c0b4d27f", size = 22596385, upload-time = "2026-07-19T11:44:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/d3/32/3a71d17c24293ae4a670d7a254565a2af60468c5a9d0ef18785197b1ee6b/helix_db_embedded-0.2.0b3-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:a9458c87dbd9112c5419a36f4dbaf12b380ceb1d0c48ca27999ecc500fbf8e92", size = 23577363, upload-time = "2026-07-19T11:45:03.691Z" }, ] [[package]] @@ -1478,15 +1378,15 @@ name = "huggingface-hub" version = "1.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", marker = "python_full_version >= '3.11'" }, - { name = "fsspec", marker = "python_full_version >= '3.11'" }, - { name = "hf-xet", marker = "(python_full_version >= '3.11' and platform_machine == 'AMD64') or (python_full_version >= '3.11' and platform_machine == 'aarch64') or (python_full_version >= '3.11' and platform_machine == 'amd64') or (python_full_version >= '3.11' and platform_machine == 'arm64') or (python_full_version >= '3.11' and platform_machine == 'x86_64')" }, - { name = "httpx", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "pyyaml", marker = "python_full_version >= '3.11'" }, - { name = "tqdm", marker = "python_full_version >= '3.11'" }, - { name = "typer", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/b6/e22bd20a25299c34b8c5922c1545a6320825b13906eb0f7298edfd034a0b/huggingface_hub-1.15.0.tar.gz", hash = "sha256:28abfdddda3927fd4de6a63cf26ab012498a2c24dae52baf150c5c6edf98a1d5", size = 784100, upload-time = "2026-05-15T11:42:52.149Z" } wheels = [ @@ -1506,29 +1406,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2e/20/96dc2387cf29a0ec75b427d62d3dde1f44c924719503babaac4c96806223/hypothesis-6.153.0-py3-none-any.whl", hash = "sha256:2aeda9bbb44ae0ee0bfa67ef744a25be05c1f804dca4eb6479c63518dc9f2900", size = 540326, upload-time = "2026-05-26T05:19:02.861Z" }, ] -[[package]] -name = "hyppo" -version = "0.5.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "autograd", marker = "python_full_version < '3.13'" }, - { name = "future", marker = "python_full_version < '3.13'" }, - { name = "numba", marker = "python_full_version < '3.13'" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "patsy", marker = "python_full_version < '3.13'" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "statsmodels", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dd/a6/0d84fe8486a1447da8bdb8ebb249d525fd8c1d0fe038bceb003c6e0513f9/hyppo-0.5.2.tar.gz", hash = "sha256:4634d15516248a43d25c241ed18beeb79bb3210360f7253693b3f154fe8c9879", size = 125115, upload-time = "2025-05-24T18:33:27.418Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/c4/d46858cfac3c0aad314a1fc378beae5c8cac499b677650a34b5a6a3d4328/hyppo-0.5.2-py3-none-any.whl", hash = "sha256:5cc18f9e158fe2cf1804c9a1e979e807118ee89a303f29dc5cb8891d92d44ef3", size = 192272, upload-time = "2025-05-24T18:33:25.904Z" }, -] - [[package]] name = "identify" version = "2.6.19" @@ -1552,7 +1429,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "python_full_version < '3.11'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -1686,15 +1563,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] -[[package]] -name = "joblib" -version = "1.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, -] - [[package]] name = "jsonschema" version = "4.26.0" @@ -1858,38 +1726,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/40/791891d4c0c4dab4c5e187c17261cedc26285fd41541577f900470a45a4d/license_expression-30.4.4-py3-none-any.whl", hash = "sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4", size = 120615, upload-time = "2025-07-22T11:13:31.217Z" }, ] -[[package]] -name = "llvmlite" -version = "0.47.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/88/a8952b6d5c21e74cbf158515b779666f692846502623e9e3c39d8e8ba25f/llvmlite-0.47.0.tar.gz", hash = "sha256:62031ce968ec74e95092184d4b0e857e444f8fdff0b8f9213707699570c33ccc", size = 193614, upload-time = "2026-03-31T18:29:53.497Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/f5/a1bde3aa8c43524b0acaf3f72fb3d80a32dd29dbb42d7dc434f84584cdcc/llvmlite-0.47.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41270b0b1310717f717cf6f2a9c68d3c43bd7905c33f003825aebc361d0d1b17", size = 37232772, upload-time = "2026-03-31T18:28:12.198Z" }, - { url = "https://files.pythonhosted.org/packages/7c/fb/76d88fc05ee1f9c1a6efe39eb493c4a727e5d1690412469017cd23bcb776/llvmlite-0.47.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f9d118bc1dd7623e0e65ca9ac485ec6dd543c3b77bc9928ddc45ebd34e1e30a7", size = 56275179, upload-time = "2026-03-31T18:28:15.725Z" }, - { url = "https://files.pythonhosted.org/packages/4d/08/29da7f36217abd56a0c389ef9a18bea47960826e691ced1a36c92c6ce93c/llvmlite-0.47.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ea5cfb04a6ab5b18e46be72b41b015975ba5980c4ddb41f1975b83e19031063", size = 55128632, upload-time = "2026-03-31T18:28:19.946Z" }, - { url = "https://files.pythonhosted.org/packages/df/f8/5e12e9ed447d65f04acf6fcf2d79cded2355640b5131a46cee4c99a5949d/llvmlite-0.47.0-cp310-cp310-win_amd64.whl", hash = "sha256:166b896a2262a2039d5fc52df5ee1659bd1ccd081183df7a2fba1b74702dd5ea", size = 38138402, upload-time = "2026-03-31T18:28:23.327Z" }, - { url = "https://files.pythonhosted.org/packages/34/0b/b9d1911cfefa61399821dfb37f486d83e0f42630a8d12f7194270c417002/llvmlite-0.47.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:74090f0dcfd6f24ebbef3f21f11e38111c4d7e6919b54c4416e1e357c3446b07", size = 37232770, upload-time = "2026-03-31T18:28:26.765Z" }, - { url = "https://files.pythonhosted.org/packages/46/27/5799b020e4cdfb25a7c951c06a96397c135efcdc21b78d853bbd9c814c7d/llvmlite-0.47.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca14f02e29134e837982497959a8e2193d6035235de1cb41a9cb2bd6da4eedbb", size = 56275177, upload-time = "2026-03-31T18:28:31.01Z" }, - { url = "https://files.pythonhosted.org/packages/7e/51/48a53fedf01cb1f3f43ef200be17ebf83c8d9a04018d3783c1a226c342c2/llvmlite-0.47.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12a69d4bb05f402f30477e21eeabe81911e7c251cecb192bed82cd83c9db10d8", size = 55128631, upload-time = "2026-03-31T18:28:36.046Z" }, - { url = "https://files.pythonhosted.org/packages/a2/50/59227d06bdc96e23322713c381af4e77420949d8cd8a042c79e0043096cc/llvmlite-0.47.0-cp311-cp311-win_amd64.whl", hash = "sha256:c37d6eb7aaabfa83ab9c2ff5b5cdb95a5e6830403937b2c588b7490724e05327", size = 38138400, upload-time = "2026-03-31T18:28:40.076Z" }, - { url = "https://files.pythonhosted.org/packages/fa/48/4b7fe0e34c169fa2f12532916133e0b219d2823b540733651b34fdac509a/llvmlite-0.47.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:306a265f408c259067257a732c8e159284334018b4083a9e35f67d19792b164f", size = 37232769, upload-time = "2026-03-31T18:28:43.735Z" }, - { url = "https://files.pythonhosted.org/packages/e6/4b/e3f2cd17822cf772a4a51a0a8080b0032e6d37b2dbe8cfb724eac4e31c52/llvmlite-0.47.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5853bf26160857c0c2573415ff4efe01c4c651e59e2c55c2a088740acfee51cd", size = 56275178, upload-time = "2026-03-31T18:28:48.342Z" }, - { url = "https://files.pythonhosted.org/packages/b6/55/a3b4a543185305a9bdf3d9759d53646ed96e55e7dfd43f53e7a421b8fbae/llvmlite-0.47.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:003bcf7fa579e14db59c1a1e113f93ab8a06b56a4be31c7f08264d1d4072d077", size = 55128632, upload-time = "2026-03-31T18:28:52.901Z" }, - { url = "https://files.pythonhosted.org/packages/2f/f5/d281ae0f79378a5a91f308ea9fdb9f9cc068fddd09629edc0725a5a8fde1/llvmlite-0.47.0-cp312-cp312-win_amd64.whl", hash = "sha256:f3079f25bdc24cd9d27c4b2b5e68f5f60c4fdb7e8ad5ee2b9b006007558f9df7", size = 38138692, upload-time = "2026-03-31T18:28:57.147Z" }, - { url = "https://files.pythonhosted.org/packages/77/6f/4615353e016799f80fa52ccb270a843c413b22361fadda2589b2922fb9b0/llvmlite-0.47.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:a3c6a735d4e1041808434f9d440faa3d78d9b4af2ee64d05a66f351883b6ceec", size = 37232771, upload-time = "2026-03-31T18:29:01.324Z" }, - { url = "https://files.pythonhosted.org/packages/31/b8/69f5565f1a280d032525878a86511eebed0645818492feeb169dfb20ae8e/llvmlite-0.47.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2699a74321189e812d476a43d6d7f652f51811e7b5aad9d9bba842a1c7927acb", size = 56275178, upload-time = "2026-03-31T18:29:05.748Z" }, - { url = "https://files.pythonhosted.org/packages/d6/da/b32cafcb926fb0ce2aa25553bf32cb8764af31438f40e2481df08884c947/llvmlite-0.47.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c6951e2b29930227963e53ee152441f0e14be92e9d4231852102d986c761e40", size = 55128632, upload-time = "2026-03-31T18:29:11.235Z" }, - { url = "https://files.pythonhosted.org/packages/46/9f/4898b44e4042c60fafcb1162dfb7014f6f15b1ec19bf29cfea6bf26df90d/llvmlite-0.47.0-cp313-cp313-win_amd64.whl", hash = "sha256:c2e9adf8698d813a9a5efb2d4370caf344dbc1e145019851fee6a6f319ba760e", size = 38138695, upload-time = "2026-03-31T18:29:15.43Z" }, - { url = "https://files.pythonhosted.org/packages/1c/d4/33c8af00f0bf6f552d74f3a054f648af2c5bc6bece97972f3bfadce4f5ec/llvmlite-0.47.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:de966c626c35c9dff5ae7bf12db25637738d0df83fc370cf793bc94d43d92d14", size = 37232773, upload-time = "2026-03-31T18:29:19.453Z" }, - { url = "https://files.pythonhosted.org/packages/64/1d/a760e993e0c0ba6db38d46b9f48f6c7dceb8ac838824997fb9e25f97bc04/llvmlite-0.47.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ddbccff2aeaff8670368340a158abefc032fe9b3ccf7d9c496639263d00151aa", size = 56275176, upload-time = "2026-03-31T18:29:24.149Z" }, - { url = "https://files.pythonhosted.org/packages/84/3b/e679bc3b29127182a7f4aa2d2e9e5bea42adb93fb840484147d59c236299/llvmlite-0.47.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a7b778a2e144fc64468fb9bf509ac1226c9813a00b4d7afea5d988c4e22fca", size = 55128631, upload-time = "2026-03-31T18:29:29.536Z" }, - { url = "https://files.pythonhosted.org/packages/be/f7/19e2a09c62809c9e63bbd14ce71fb92c6ff7b7b3045741bb00c781efc3c9/llvmlite-0.47.0-cp314-cp314-win_amd64.whl", hash = "sha256:694e3c2cdc472ed2bd8bd4555ca002eec4310961dd58ef791d508f57b5cc4c94", size = 39153826, upload-time = "2026-03-31T18:29:33.681Z" }, - { url = "https://files.pythonhosted.org/packages/40/a1/581a8c707b5e80efdbbe1dd94527404d33fe50bceb71f39d5a7e11bd57b7/llvmlite-0.47.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:92ec8a169a20b473c1c54d4695e371bde36489fc1efa3688e11e99beba0abf9c", size = 37232772, upload-time = "2026-03-31T18:29:37.952Z" }, - { url = "https://files.pythonhosted.org/packages/11/03/16090dd6f74ba2b8b922276047f15962fbeea0a75d5601607edb301ba945/llvmlite-0.47.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbd800edd3b20bc141521f7fd45a6185a5b84109aa6855134e81397ffe72b", size = 56275178, upload-time = "2026-03-31T18:29:42.58Z" }, - { url = "https://files.pythonhosted.org/packages/f5/cb/0abf1dd4c5286a95ffe0c1d8c67aec06b515894a0dd2ac97f5e27b82ab0b/llvmlite-0.47.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6725179b89f03b17dabe236ff3422cb8291b4c1bf40af152826dfd34e350ae8", size = 55128632, upload-time = "2026-03-31T18:29:46.939Z" }, - { url = "https://files.pythonhosted.org/packages/4f/79/d3bbab197e86e0ff4f9c07122895b66a3e0d024247fcff7f12c473cb36d9/llvmlite-0.47.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6842cf6f707ec4be3d985a385ad03f72b2d724439e118fcbe99b2929964f0453", size = 39153839, upload-time = "2026-03-31T18:29:51.004Z" }, -] - [[package]] name = "lxml" version = "6.1.0" @@ -2110,7 +1946,7 @@ wheels = [ [[package]] name = "mcp" -version = "1.27.1" +version = "1.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2128,9 +1964,9 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/83/d1efe7c2980d8a3afa476f4e3d42d53dd54c0ab94c27bee5d755b45c8b73/mcp-1.27.1.tar.gz", hash = "sha256:0f47e1820f8f8f941466b39749eb1d1839a04caddca2bc60e9d46e8a99914924", size = 608458, upload-time = "2026-05-08T16:50:12.601Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/77/9450b8f251a13affb6281997d0523c4615f8a8b35d0b21ff30db3a5aac9d/mcp-1.28.1.tar.gz", hash = "sha256:d51e36a5f5644faea4f85ea649bfffa6bc6c26770d42798ad6a3de3d2ba69683", size = 638501, upload-time = "2026-06-26T12:57:29.093Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/73/42d9596facebdb533b7f0b86c1b0364ef350d1f8ba78b1052e8a58b48b65/mcp-1.27.1-py3-none-any.whl", hash = "sha256:1af3c4203b329430fde7a87b4fcb6392a041f5cb851fd68fc674016ab4e7c06f", size = 216260, upload-time = "2026-05-08T16:50:10.547Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5e/d118fce19f87a2e7d8101c35c8ae0ec289098a4df0ff244cec23e415aca0/mcp-1.28.1-py3-none-any.whl", hash = "sha256:2726bca5e7193f61c5dde8b12500a6de2d9acf6d1a1c0be9e8c2e706437991df", size = 222620, upload-time = "2026-06-26T12:57:27.218Z" }, ] [[package]] @@ -2227,38 +2063,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/cf/1c3795866cefaac6e648d4e98c373cafd97810f6e317c307371007ab4abb/neo4j-6.2.0-py3-none-any.whl", hash = "sha256:b87abdd13a5cc2e3bd51026926c2f20ac38fa3febe98c340520dce19e97388d0", size = 327824, upload-time = "2026-05-04T07:35:39.604Z" }, ] -[[package]] -name = "networkx" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263, upload-time = "2024-10-21T12:39:36.247Z" }, -] - -[[package]] -name = "networkx" -version = "3.6.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, -] - [[package]] name = "nodeenv" version = "1.10.0" @@ -2274,42 +2078,6 @@ version = "4.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/53/db/b7a344ad688cd6d8547746869f904b105674ff529a24fa5a3d7bdd95560a/nuitka-4.1.tar.gz", hash = "sha256:99092d26f5f8d5264186924451f7df5872bf6a922297062ace2798ecec7cfa0f", size = 4543258, upload-time = "2026-05-11T07:16:35.483Z" } -[[package]] -name = "numba" -version = "0.65.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llvmlite", marker = "python_full_version < '3.13'" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f6/c5/db2ac3685833d626c0dcae6bd2330cd68433e1fd248d15f70998160d3ad7/numba-0.65.1.tar.gz", hash = "sha256:19357146c32fe9ed25059ab915e8465fb13951cf6b0aace3826b76886373ab23", size = 2765600, upload-time = "2026-04-24T02:02:56.551Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/1b/3c5a7daf683a95465bf23504bcd1a2d5db8cd5e5e276ca87505d020dffe9/numba-0.65.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:9d993ed0a257aa4116e6f553f114004bcfdee540c7276ab8ea48f650d514c452", size = 2680870, upload-time = "2026-04-24T02:02:10.623Z" }, - { url = "https://files.pythonhosted.org/packages/0f/a4/1831836814018a898e7d252aebe09c0f3ce1f26d145b68264b4ae0be6822/numba-0.65.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5f098109f361681e57295f7e84d8ab2426902539a141811de0703ace52826981", size = 3739780, upload-time = "2026-04-24T02:02:13.097Z" }, - { url = "https://files.pythonhosted.org/packages/9c/1b/a813ddc81def09e257d2b1f67521982ce4b06204a87268796ffc8187271c/numba-0.65.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:973fd8173f2312815e6b7aaae887c4ce8a817eeff46a4f8840b828305b75bc95", size = 3446722, upload-time = "2026-04-24T02:02:15.083Z" }, - { url = "https://files.pythonhosted.org/packages/09/52/ee1d8b3becda384fe0552221641e05aa668a35e8a77470db4db7f6475000/numba-0.65.1-cp310-cp310-win_amd64.whl", hash = "sha256:c63aa0c4193694026452da55d0ef9d85156c1a7a333454c103bb30dec81b7bf8", size = 2747539, upload-time = "2026-04-24T02:02:16.79Z" }, - { url = "https://files.pythonhosted.org/packages/96/b3/650500c2eab4534d98e9166f4298e0f3c69c742afdf24e6eabccd1f16ad8/numba-0.65.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:7020d74b19cdb8cff16506542fdd510756e28c5e7f3bd0b7f574f0f42272fcd9", size = 2680563, upload-time = "2026-04-24T02:02:18.414Z" }, - { url = "https://files.pythonhosted.org/packages/44/0b/0615dbedb98f5b32a35a53290fbdc6e22306968109278d7e58df82d7a9f6/numba-0.65.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f80ed83774b5173abd6581cd8d2165d1d38e13d2e5c8155c0c0b421784745420", size = 3745018, upload-time = "2026-04-24T02:02:20.252Z" }, - { url = "https://files.pythonhosted.org/packages/49/aa/4361698f35bf63bff67dfe6c90493731177f48ede954f77b0588731537bc/numba-0.65.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7ed425a43b0a5f9772f2f4e2dd0bbd12eabecae1af0b24efcfd4e053f012aac6", size = 3450962, upload-time = "2026-04-24T02:02:22.449Z" }, - { url = "https://files.pythonhosted.org/packages/bd/9a/af61ec03b3116c161fd7a06b9e8a265729a8718458333e8ffbb06d9a3978/numba-0.65.1-cp311-cp311-win_amd64.whl", hash = "sha256:df40a5028a975b9ea66f6a2a3f7abbdbd541a863070e34ed367aff21141248e4", size = 2747417, upload-time = "2026-04-24T02:02:24.43Z" }, - { url = "https://files.pythonhosted.org/packages/57/bc/76f8f8c5cf9adee47fdb7bbb03be8900f76f902d451d7477cf12b845e1de/numba-0.65.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ac3f1e77c352dd0ea9712732c2d8f9ca507717435eec5b5013bf138ac33c4a08", size = 2681371, upload-time = "2026-04-24T02:02:26.105Z" }, - { url = "https://files.pythonhosted.org/packages/69/47/a415af0283e4db0398104c6d1c11c9861a98dc67a7aa442a7769ed5d6196/numba-0.65.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:52bc6f3ceb8fcaff9b2ae26b4c6b1e9fee39db8d355534c0fe4f39a901246b84", size = 3802467, upload-time = "2026-04-24T02:02:27.712Z" }, - { url = "https://files.pythonhosted.org/packages/46/36/246f73ec99cfeab2f2cb2ce7d4218766cc36a2da418901223f4f4da9c813/numba-0.65.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90ca10b3463bae0bd70589726fe3c77d01d6b5fc86bee54bcdf9fb6b47c28977", size = 3502628, upload-time = "2026-04-24T02:02:29.763Z" }, - { url = "https://files.pythonhosted.org/packages/db/9e/3c679b2ee078425b9e99a91e44f8d132a6830d8ccce5227bc5e9181aeed8/numba-0.65.1-cp312-cp312-win_amd64.whl", hash = "sha256:5971c632be2a2351500431f46213821dba8d02b18a9f7d02fd36bd2743e41a6a", size = 2750611, upload-time = "2026-04-24T02:02:31.477Z" }, - { url = "https://files.pythonhosted.org/packages/79/37/14a4579049c1eb673afd0de0cb4842982acd55b9ce2643e763db858bcea0/numba-0.65.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:1735c15c1134a5108b4d6a5c77fc0947924ea066a738dc09a52008c13df9cad3", size = 2681344, upload-time = "2026-04-24T02:02:33.65Z" }, - { url = "https://files.pythonhosted.org/packages/a0/22/b8d873f6466b20aa563fc9b33acd48dec89a07803ddaa2f1c8ca1cd33126/numba-0.65.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c09f49117ef255e1f1c6dad0c7a1ed39868243862a73be5706793241a3755f1b", size = 3810619, upload-time = "2026-04-24T02:02:36.041Z" }, - { url = "https://files.pythonhosted.org/packages/62/08/e16a8b5d9a018962ebb5c66be662317cde32b9f5dab08441f90bed5522fb/numba-0.65.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:594a8680b3fadac99e97e489b1fd89007177e5336713745c3b769528c635a464", size = 3509783, upload-time = "2026-04-24T02:02:38.245Z" }, - { url = "https://files.pythonhosted.org/packages/fd/a5/03c970d57f4c1741354837353ce39fb5206952ae1dba8922d29c86f64805/numba-0.65.1-cp313-cp313-win_amd64.whl", hash = "sha256:85be74c0d036842699a30058f82fb88fc5ffdc59f7615cab5792ea92914c9b62", size = 2750534, upload-time = "2026-04-24T02:02:39.903Z" }, - { url = "https://files.pythonhosted.org/packages/4f/2e/8aed9b726d9ba5f11ad287645fd479e88278db3060a25cb1225d730eb2b7/numba-0.65.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:33f5eb68eb1c843511615d14663ce60258525d6a4c65ab040e2c2b0c4cf17450", size = 2681554, upload-time = "2026-04-24T02:02:41.812Z" }, - { url = "https://files.pythonhosted.org/packages/87/96/f3eb235fafa82a34e2ab5dd7dc9ffff998ebf5f0bbc23fa56a96aeb44da6/numba-0.65.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71e73029bf53a62cc6afcf96be4bd942290d8b4c55f0a454fb536158115790f7", size = 3779602, upload-time = "2026-04-24T02:02:43.726Z" }, - { url = "https://files.pythonhosted.org/packages/09/90/b0f09b48752d23640b8284f22aa597737e8adaddc7fbfacc4708b7f73a4c/numba-0.65.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a07635e0be926b9bdbffb09137c230fb13f6ec0e564914ba937cee12ce3eb35", size = 3479532, upload-time = "2026-04-24T02:02:45.427Z" }, - { url = "https://files.pythonhosted.org/packages/56/46/3f7fc04fb853559e74b210e0b62c19974ec844cefec611f9e535f4da3761/numba-0.65.1-cp314-cp314-win_amd64.whl", hash = "sha256:2a20fcdabdefbdacf88d85caf70c3b18c4bcb7ebb8f82e6a19486383dd26ab63", size = 2752637, upload-time = "2026-04-24T02:02:47.664Z" }, - { url = "https://files.pythonhosted.org/packages/81/7b/c1a341a9067367778f4152a5f01061cf281fb09582c92c510ec4918cabf6/numba-0.65.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:548dd4b3a4508d5062768d1514b2cd7b015f9a25ec7af651c50dee243965e652", size = 2684600, upload-time = "2026-04-24T02:02:49.653Z" }, - { url = "https://files.pythonhosted.org/packages/03/36/98ddbcf3e4f04a6dd07e1c67249955920579ba4af6bb6868e3088f4ed282/numba-0.65.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78abc28feff2c2ff8307fff3975b6438352759c9acb797ecd6b1fb6e7e39e31d", size = 3817198, upload-time = "2026-04-24T02:02:51.266Z" }, - { url = "https://files.pythonhosted.org/packages/a3/83/0dad21057ece5a835599f5d24099b091703995e23dbbf894f259e91c010b/numba-0.65.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee7676cb389555805f9b9a1840cbcd1ea6c8bd5376ab6918e3a29c5ea1dbda20", size = 3533862, upload-time = "2026-04-24T02:02:52.987Z" }, - { url = "https://files.pythonhosted.org/packages/32/36/8be7118ffd4c8440881046eac3d0982cc5ab42909508cf5d67024d62a2e4/numba-0.65.1-cp314-cp314t-win_amd64.whl", hash = "sha256:20609346e3bd75204950dcbbfe383a8d7dbf4902f442aedbf00f97fef4aa8f38", size = 2758237, upload-time = "2026-04-24T02:02:54.612Z" }, -] - [[package]] name = "numpy" version = "1.26.4" @@ -2440,11 +2208,11 @@ name = "onnxruntime" version = "1.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "flatbuffers", marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, + { name = "flatbuffers" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "protobuf", marker = "python_full_version >= '3.11'" }, + { name = "packaging" }, + { name = "protobuf" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/d4/81/29a9eb470994a75eb7b3ccf32be314d7c66675a00ac7b50294816cc2db27/onnxruntime-1.26.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ee1109ef4ef27cad90e823399e61e03b3c6c7bfe0fb820b4baf3678c15be8b3c", size = 18005108, upload-time = "2026-05-08T19:08:11.728Z" }, @@ -2522,135 +2290,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] -[[package]] -name = "pandas" -version = "2.3.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/f7/f425a00df4fcc22b292c6895c6831c0c8ae1d9fac1e024d16f98a9ce8749/pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c", size = 11555763, upload-time = "2025-09-29T23:16:53.287Z" }, - { url = "https://files.pythonhosted.org/packages/13/4f/66d99628ff8ce7857aca52fed8f0066ce209f96be2fede6cef9f84e8d04f/pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a", size = 10801217, upload-time = "2025-09-29T23:17:04.522Z" }, - { url = "https://files.pythonhosted.org/packages/1d/03/3fc4a529a7710f890a239cc496fc6d50ad4a0995657dccc1d64695adb9f4/pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1", size = 12148791, upload-time = "2025-09-29T23:17:18.444Z" }, - { url = "https://files.pythonhosted.org/packages/40/a8/4dac1f8f8235e5d25b9955d02ff6f29396191d4e665d71122c3722ca83c5/pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838", size = 12769373, upload-time = "2025-09-29T23:17:35.846Z" }, - { url = "https://files.pythonhosted.org/packages/df/91/82cc5169b6b25440a7fc0ef3a694582418d875c8e3ebf796a6d6470aa578/pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250", size = 13200444, upload-time = "2025-09-29T23:17:49.341Z" }, - { url = "https://files.pythonhosted.org/packages/10/ae/89b3283800ab58f7af2952704078555fa60c807fff764395bb57ea0b0dbd/pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4", size = 13858459, upload-time = "2025-09-29T23:18:03.722Z" }, - { url = "https://files.pythonhosted.org/packages/85/72/530900610650f54a35a19476eca5104f38555afccda1aa11a92ee14cb21d/pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826", size = 11346086, upload-time = "2025-09-29T23:18:18.505Z" }, - { url = "https://files.pythonhosted.org/packages/c1/fa/7ac648108144a095b4fb6aa3de1954689f7af60a14cf25583f4960ecb878/pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523", size = 11578790, upload-time = "2025-09-29T23:18:30.065Z" }, - { url = "https://files.pythonhosted.org/packages/9b/35/74442388c6cf008882d4d4bdfc4109be87e9b8b7ccd097ad1e7f006e2e95/pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45", size = 10833831, upload-time = "2025-09-29T23:38:56.071Z" }, - { url = "https://files.pythonhosted.org/packages/fe/e4/de154cbfeee13383ad58d23017da99390b91d73f8c11856f2095e813201b/pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66", size = 12199267, upload-time = "2025-09-29T23:18:41.627Z" }, - { url = "https://files.pythonhosted.org/packages/bf/c9/63f8d545568d9ab91476b1818b4741f521646cbdd151c6efebf40d6de6f7/pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b", size = 12789281, upload-time = "2025-09-29T23:18:56.834Z" }, - { url = "https://files.pythonhosted.org/packages/f2/00/a5ac8c7a0e67fd1a6059e40aa08fa1c52cc00709077d2300e210c3ce0322/pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791", size = 13240453, upload-time = "2025-09-29T23:19:09.247Z" }, - { url = "https://files.pythonhosted.org/packages/27/4d/5c23a5bc7bd209231618dd9e606ce076272c9bc4f12023a70e03a86b4067/pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151", size = 13890361, upload-time = "2025-09-29T23:19:25.342Z" }, - { url = "https://files.pythonhosted.org/packages/8e/59/712db1d7040520de7a4965df15b774348980e6df45c129b8c64d0dbe74ef/pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c", size = 11348702, upload-time = "2025-09-29T23:19:38.296Z" }, - { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" }, - { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" }, - { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, - { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, - { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, - { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, - { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" }, - { url = "https://files.pythonhosted.org/packages/cd/4b/18b035ee18f97c1040d94debd8f2e737000ad70ccc8f5513f4eefad75f4b/pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713", size = 11544671, upload-time = "2025-09-29T23:21:05.024Z" }, - { url = "https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8", size = 10680807, upload-time = "2025-09-29T23:21:15.979Z" }, - { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872, upload-time = "2025-09-29T23:21:27.165Z" }, - { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371, upload-time = "2025-09-29T23:21:40.532Z" }, - { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333, upload-time = "2025-09-29T23:21:55.77Z" }, - { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120, upload-time = "2025-09-29T23:22:10.109Z" }, - { url = "https://files.pythonhosted.org/packages/4f/c7/e54682c96a895d0c808453269e0b5928a07a127a15704fedb643e9b0a4c8/pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee", size = 10993991, upload-time = "2025-09-29T23:25:04.889Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ca/3f8d4f49740799189e1395812f3bf23b5e8fc7c190827d55a610da72ce55/pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5", size = 12048227, upload-time = "2025-09-29T23:22:24.343Z" }, - { url = "https://files.pythonhosted.org/packages/0e/5a/f43efec3e8c0cc92c4663ccad372dbdff72b60bdb56b2749f04aa1d07d7e/pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21", size = 11411056, upload-time = "2025-09-29T23:22:37.762Z" }, - { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189, upload-time = "2025-09-29T23:22:51.688Z" }, - { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912, upload-time = "2025-09-29T23:23:05.042Z" }, - { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160, upload-time = "2025-09-29T23:23:28.57Z" }, - { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, - { url = "https://files.pythonhosted.org/packages/04/fd/74903979833db8390b73b3a8a7d30d146d710bd32703724dd9083950386f/pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0", size = 11540635, upload-time = "2025-09-29T23:25:52.486Z" }, - { url = "https://files.pythonhosted.org/packages/21/00/266d6b357ad5e6d3ad55093a7e8efc7dd245f5a842b584db9f30b0f0a287/pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593", size = 10759079, upload-time = "2025-09-29T23:26:33.204Z" }, - { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" }, - { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" }, - { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" }, - { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" }, - { url = "https://files.pythonhosted.org/packages/a6/3d/124ac75fcd0ecc09b8fdccb0246ef65e35b012030defb0e0eba2cbbbe948/pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5", size = 11109071, upload-time = "2025-09-29T23:32:27.484Z" }, - { url = "https://files.pythonhosted.org/packages/89/9c/0e21c895c38a157e0faa1fb64587a9226d6dd46452cac4532d80c3c4a244/pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec", size = 12048504, upload-time = "2025-09-29T23:29:31.47Z" }, - { url = "https://files.pythonhosted.org/packages/d7/82/b69a1c95df796858777b68fbe6a81d37443a33319761d7c652ce77797475/pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7", size = 11410702, upload-time = "2025-09-29T23:29:54.591Z" }, - { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" }, - { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" }, - { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" }, - { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" }, -] - -[[package]] -name = "pandas" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32')" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/16/b5c76b838fd9bf6ce84d3a53346b8874ec05c5f0040d75ef2c320100cd2a/pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98", size = 10338495, upload-time = "2026-05-11T18:52:11.558Z" }, - { url = "https://files.pythonhosted.org/packages/5a/b0/a4ffc4ae74d2d822200dcc46898987d8eb6032d1e2b219cae39da6f5cbcc/pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639", size = 9938250, upload-time = "2026-05-11T18:52:17.005Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b2/3323601a52caee42c019e370090ca4544b241437240ca04f786cce82b0cf/pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2", size = 10770558, upload-time = "2026-05-11T18:52:19.865Z" }, - { url = "https://files.pythonhosted.org/packages/32/f1/bbecd2f867b97abebe0f9b53d750f862251b40337e061b36676ded3d920f/pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27", size = 11274611, upload-time = "2026-05-11T18:52:22.622Z" }, - { url = "https://files.pythonhosted.org/packages/7f/4f/eafabf2d5fae5adf143b4d18d3706c5efdc368a7c4eb1ee8a3eddabbd0f6/pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824", size = 11784670, upload-time = "2026-05-11T18:52:25.4Z" }, - { url = "https://files.pythonhosted.org/packages/49/44/1eb20389301b57b19cc099a1c2f662501f72f08a65f912d05822613c1532/pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938", size = 12353708, upload-time = "2026-05-11T18:52:28.139Z" }, - { url = "https://files.pythonhosted.org/packages/eb/62/c321f13b5ba1819fc8dca456c7fce578da2dcfecff1abbf0eaddf8406c0f/pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea", size = 9907609, upload-time = "2026-05-11T18:52:30.982Z" }, - { url = "https://files.pythonhosted.org/packages/53/85/1b7f563ebc6357c27233a02a96b589bcce1fa9c6eb89fb4f0e56421d277e/pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a", size = 9165596, upload-time = "2026-05-11T18:52:33.334Z" }, - { url = "https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09", size = 10357846, upload-time = "2026-05-11T18:52:36.143Z" }, - { url = "https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4", size = 9899550, upload-time = "2026-05-11T18:52:38.976Z" }, - { url = "https://files.pythonhosted.org/packages/31/a8/fa2535168fffcedf67f4f6de28d2dd903a747ca7c8ea6989451aaeb3a92f/pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c", size = 10412965, upload-time = "2026-05-11T18:52:41.915Z" }, - { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a4/2eb28f2fccb4ced4a2c79ab2a5dee9ade1ebf44922ebad6fea158c9f95d4/pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf", size = 11422824, upload-time = "2026-05-11T18:52:48.058Z" }, - { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" }, - { url = "https://files.pythonhosted.org/packages/b9/c5/fc1b368f303087d20e8c9bf3d6ceb186263cfac0ade735cd938538bea839/pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc", size = 9755463, upload-time = "2026-05-11T18:52:53.386Z" }, - { url = "https://files.pythonhosted.org/packages/86/bd/fda8f9705b1b09c6ebe14bfc0fa0e4ec8584d54ea673628f157ff55131af/pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49", size = 9066158, upload-time = "2026-05-11T18:52:56.038Z" }, - { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, - { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, - { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, - { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, - { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, - { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, - { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, - { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, - { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, - { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, - { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, - { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, - { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, - { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, - { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, - { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, - { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, - { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, - { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, - { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, - { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, - { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, - { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, - { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, - { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, - { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, - { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, - { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, - { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, - { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, -] - [[package]] name = "patchelf" version = "0.17.2.4" @@ -2667,114 +2306,98 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/e2/975d4bdb418f942b53e6187b95bd9e0d5e0488b7bc214685a1e43e2c2751/patchelf-0.17.2.4-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:7076d9e127230982e20a81a6e2358d3343004667ba510d9f822d4fdee29b0d71", size = 508281, upload-time = "2025-07-23T21:16:30.865Z" }, ] -[[package]] -name = "patsy" -version = "1.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/be/44/ed13eccdd0519eff265f44b670d46fbb0ec813e2274932dc1c0e48520f7d/patsy-1.0.2.tar.gz", hash = "sha256:cdc995455f6233e90e22de72c37fcadb344e7586fb83f06696f54d92f8ce74c0", size = 399942, upload-time = "2025-10-20T16:17:37.535Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/70/ba4b949bdc0490ab78d545459acd7702b211dfccf7eb89bbc1060f52818d/patsy-1.0.2-py2.py3-none-any.whl", hash = "sha256:37bfddbc58fcf0362febb5f54f10743f8b21dd2aa73dec7e7ef59d1b02ae668a", size = 233301, upload-time = "2025-10-20T16:17:36.563Z" }, -] - [[package]] name = "pillow" -version = "12.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/aa/d0b28e1c811cd4d5f5c2bfe2e022292bd255ae5744a3b9ac7d6c8f72dd75/pillow-12.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:a4e8f36e677d3336f35089648c8955c51c6d386a13cf6ee9c189c5f5bd713a9f", size = 5354355, upload-time = "2026-04-01T14:42:15.402Z" }, - { url = "https://files.pythonhosted.org/packages/27/8e/1d5b39b8ae2bd7650d0c7b6abb9602d16043ead9ebbfef4bc4047454da2a/pillow-12.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e589959f10d9824d39b350472b92f0ce3b443c0a3442ebf41c40cb8361c5b97", size = 4695871, upload-time = "2026-04-01T14:42:18.234Z" }, - { url = "https://files.pythonhosted.org/packages/f0/c5/dcb7a6ca6b7d3be41a76958e90018d56c8462166b3ef223150360850c8da/pillow-12.2.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a52edc8bfff4429aaabdf4d9ee0daadbbf8562364f940937b941f87a4290f5ff", size = 6269734, upload-time = "2026-04-01T14:42:20.608Z" }, - { url = "https://files.pythonhosted.org/packages/ea/f1/aa1bb13b2f4eba914e9637893c73f2af8e48d7d4023b9d3750d4c5eb2d0c/pillow-12.2.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:975385f4776fafde056abb318f612ef6285b10a1f12b8570f3647ad0d74b48ec", size = 8076080, upload-time = "2026-04-01T14:42:23.095Z" }, - { url = "https://files.pythonhosted.org/packages/a1/2a/8c79d6a53169937784604a8ae8d77e45888c41537f7f6f65ed1f407fe66d/pillow-12.2.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd9c0c7a0c681a347b3194c500cb1e6ca9cab053ea4d82a5cf45b6b754560136", size = 6382236, upload-time = "2026-04-01T14:42:25.82Z" }, - { url = "https://files.pythonhosted.org/packages/b5/42/bbcb6051030e1e421d103ce7a8ecadf837aa2f39b8f82ef1a8d37c3d4ebc/pillow-12.2.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88d387ff40b3ff7c274947ed3125dedf5262ec6919d83946753b5f3d7c67ea4c", size = 7070220, upload-time = "2026-04-01T14:42:28.68Z" }, - { url = "https://files.pythonhosted.org/packages/3f/e1/c2a7d6dd8cfa6b231227da096fd2d58754bab3603b9d73bf609d3c18b64f/pillow-12.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:51c4167c34b0d8ba05b547a3bb23578d0ba17b80a5593f93bd8ecb123dd336a3", size = 6493124, upload-time = "2026-04-01T14:42:31.579Z" }, - { url = "https://files.pythonhosted.org/packages/5f/41/7c8617da5d32e1d2f026e509484fdb6f3ad7efaef1749a0c1928adbb099e/pillow-12.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34c0d99ecccea270c04882cb3b86e7b57296079c9a4aff88cb3b33563d95afaa", size = 7194324, upload-time = "2026-04-01T14:42:34.615Z" }, - { url = "https://files.pythonhosted.org/packages/2d/de/a777627e19fd6d62f84070ee1521adde5eeda4855b5cf60fe0b149118bca/pillow-12.2.0-cp310-cp310-win32.whl", hash = "sha256:b85f66ae9eb53e860a873b858b789217ba505e5e405a24b85c0464822fe88032", size = 6376363, upload-time = "2026-04-01T14:42:37.19Z" }, - { url = "https://files.pythonhosted.org/packages/e7/34/fc4cb5204896465842767b96d250c08410f01f2f28afc43b257de842eed5/pillow-12.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:673aa32138f3e7531ccdbca7b3901dba9b70940a19ccecc6a37c77d5fdeb05b5", size = 7083523, upload-time = "2026-04-01T14:42:39.62Z" }, - { url = "https://files.pythonhosted.org/packages/2d/a0/32852d36bc7709f14dc3f64f929a275e958ad8c19a6deba9610d458e28b3/pillow-12.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:3e080565d8d7c671db5802eedfb438e5565ffa40115216eabb8cd52d0ecce024", size = 2463318, upload-time = "2026-04-01T14:42:42.063Z" }, - { url = "https://files.pythonhosted.org/packages/68/e1/748f5663efe6edcfc4e74b2b93edfb9b8b99b67f21a854c3ae416500a2d9/pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab", size = 5354347, upload-time = "2026-04-01T14:42:44.255Z" }, - { url = "https://files.pythonhosted.org/packages/47/a1/d5ff69e747374c33a3b53b9f98cca7889fce1fd03d79cdc4e1bccc6c5a87/pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65", size = 4695873, upload-time = "2026-04-01T14:42:46.452Z" }, - { url = "https://files.pythonhosted.org/packages/df/21/e3fbdf54408a973c7f7f89a23b2cb97a7ef30c61ab4142af31eee6aebc88/pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7", size = 6280168, upload-time = "2026-04-01T14:42:49.228Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f1/00b7278c7dd52b17ad4329153748f87b6756ec195ff786c2bdf12518337d/pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e", size = 8088188, upload-time = "2026-04-01T14:42:51.735Z" }, - { url = "https://files.pythonhosted.org/packages/ad/cf/220a5994ef1b10e70e85748b75649d77d506499352be135a4989c957b701/pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705", size = 6394401, upload-time = "2026-04-01T14:42:54.343Z" }, - { url = "https://files.pythonhosted.org/packages/e9/bd/e51a61b1054f09437acfbc2ff9106c30d1eb76bc1453d428399946781253/pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176", size = 7079655, upload-time = "2026-04-01T14:42:56.954Z" }, - { url = "https://files.pythonhosted.org/packages/6b/3d/45132c57d5fb4b5744567c3817026480ac7fc3ce5d4c47902bc0e7f6f853/pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b", size = 6503105, upload-time = "2026-04-01T14:42:59.847Z" }, - { url = "https://files.pythonhosted.org/packages/7d/2e/9df2fc1e82097b1df3dce58dc43286aa01068e918c07574711fcc53e6fb4/pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909", size = 7203402, upload-time = "2026-04-01T14:43:02.664Z" }, - { url = "https://files.pythonhosted.org/packages/bd/2e/2941e42858ebb67e50ae741473de81c2984e6eff7b397017623c676e2e8d/pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808", size = 6378149, upload-time = "2026-04-01T14:43:05.274Z" }, - { url = "https://files.pythonhosted.org/packages/69/42/836b6f3cd7f3e5fa10a1f1a5420447c17966044c8fbf589cc0452d5502db/pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60", size = 7082626, upload-time = "2026-04-01T14:43:08.557Z" }, - { url = "https://files.pythonhosted.org/packages/c2/88/549194b5d6f1f494b485e493edc6693c0a16f4ada488e5bd974ed1f42fad/pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe", size = 2463531, upload-time = "2026-04-01T14:43:10.743Z" }, - { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, - { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, - { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, - { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, - { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, - { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, - { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, - { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, - { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, - { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, - { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, - { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, - { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, - { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, - { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, - { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, - { url = "https://files.pythonhosted.org/packages/23/c4/7349421080b12fb35414607b8871e9534546c128a11965fd4a7002ccfbee/pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e", size = 6375896, upload-time = "2026-04-01T14:44:11.197Z" }, - { url = "https://files.pythonhosted.org/packages/3f/82/8a3739a5e470b3c6cbb1d21d315800d8e16bff503d1f16b03a4ec3212786/pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b", size = 7081266, upload-time = "2026-04-01T14:44:13.947Z" }, - { url = "https://files.pythonhosted.org/packages/c3/25/f968f618a062574294592f668218f8af564830ccebdd1fa6200f598e65c5/pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06", size = 2463508, upload-time = "2026-04-01T14:44:16.312Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, - { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, - { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, - { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, - { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, - { url = "https://files.pythonhosted.org/packages/36/a3/f9a77144231fb8d40ee27107b4463e205fa4677e2ca2548e14da5cf18dce/pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4", size = 7115667, upload-time = "2026-04-01T14:44:32.773Z" }, - { url = "https://files.pythonhosted.org/packages/c1/fc/ac4ee3041e7d5a565e1c4fd72a113f03b6394cc72ab7089d27608f8aaccb/pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4", size = 6538966, upload-time = "2026-04-01T14:44:35.252Z" }, - { url = "https://files.pythonhosted.org/packages/c0/a8/27fb307055087f3668f6d0a8ccb636e7431d56ed0750e07a60547b1e083e/pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea", size = 7238241, upload-time = "2026-04-01T14:44:37.875Z" }, - { url = "https://files.pythonhosted.org/packages/ad/4b/926ab182c07fccae9fcb120043464e1ff1564775ec8864f21a0ebce6ac25/pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24", size = 6379592, upload-time = "2026-04-01T14:44:40.336Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c4/f9e476451a098181b30050cc4c9a3556b64c02cf6497ea421ac047e89e4b/pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98", size = 7085542, upload-time = "2026-04-01T14:44:43.251Z" }, - { url = "https://files.pythonhosted.org/packages/00/a4/285f12aeacbe2d6dc36c407dfbbe9e96d4a80b0fb710a337f6d2ad978c75/pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453", size = 2465765, upload-time = "2026-04-01T14:44:45.996Z" }, - { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, - { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, - { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, - { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, - { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, - { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, - { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, - { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, - { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, - { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, - { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, - { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, - { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, - { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, - { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, - { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, - { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, - { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, - { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, - { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, - { url = "https://files.pythonhosted.org/packages/4e/b7/2437044fb910f499610356d1352e3423753c98e34f915252aafecc64889f/pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f", size = 5273969, upload-time = "2026-04-01T14:45:55.538Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f4/8316e31de11b780f4ac08ef3654a75555e624a98db1056ecb2122d008d5a/pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d", size = 4659674, upload-time = "2026-04-01T14:45:58.093Z" }, - { url = "https://files.pythonhosted.org/packages/d4/37/664fca7201f8bb2aa1d20e2c3d5564a62e6ae5111741966c8319ca802361/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f", size = 5288479, upload-time = "2026-04-01T14:46:01.141Z" }, - { url = "https://files.pythonhosted.org/packages/49/62/5b0ed78fce87346be7a5cfcfaaad91f6a1f98c26f86bdbafa2066c647ef6/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e", size = 7032230, upload-time = "2026-04-01T14:46:03.874Z" }, - { url = "https://files.pythonhosted.org/packages/c3/28/ec0fc38107fc32536908034e990c47914c57cd7c5a3ece4d8d8f7ffd7e27/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0", size = 5355404, upload-time = "2026-04-01T14:46:06.33Z" }, - { url = "https://files.pythonhosted.org/packages/5e/8b/51b0eddcfa2180d60e41f06bd6d0a62202b20b59c68f5a132e615b75aecf/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1", size = 6002215, upload-time = "2026-04-01T14:46:08.83Z" }, - { url = "https://files.pythonhosted.org/packages/bc/60/5382c03e1970de634027cee8e1b7d39776b778b81812aaf45b694dfe9e28/pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e", size = 7080946, upload-time = "2026-04-01T14:46:11.734Z" }, +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/c2/669d88644cddb1485bd9534e63e8cf476c8e51cb3c3a1297677023505c0e/pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a", size = 5392418, upload-time = "2026-07-01T11:53:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ba/3762f376a2948e3036488d773a146e0ae6ecc2ca03ac20e2615bd0b2ba02/pillow-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7", size = 4785287, upload-time = "2026-07-01T11:53:29.761Z" }, + { url = "https://files.pythonhosted.org/packages/07/50/b5d688cc9c52d4482f3d5bcab6ce20bc2a74a85d2343841c907444a3be2c/pillow-12.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f", size = 6253754, upload-time = "2026-07-01T11:53:32.298Z" }, + { url = "https://files.pythonhosted.org/packages/4e/89/36f4cd76cf4baf05c50ababb976249153f18c959171c7f6ba09a6f217260/pillow-12.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec", size = 6925605, upload-time = "2026-07-01T11:53:34.487Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c0/4de58cf6633b9e3a6061ef4be6fb91fc3c90b812ece886f531e3c523d777/pillow-12.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468", size = 6327788, upload-time = "2026-07-01T11:53:36.433Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/14d53682a19550dbbaf3b598f807d5457646c510805a44c7d7891cd1cd1a/pillow-12.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed", size = 7036288, upload-time = "2026-07-01T11:53:38.712Z" }, + { url = "https://files.pythonhosted.org/packages/38/1d/36279e3c77efe034e4cc2b0393ee74ffdb5a62391dacbf9b916154f5f0b8/pillow-12.3.0-cp310-cp310-win32.whl", hash = "sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1", size = 6472396, upload-time = "2026-07-01T11:53:40.781Z" }, + { url = "https://files.pythonhosted.org/packages/48/7c/8fa0039574c476d7c6fa57dd7c32a130436877c6ec1e5ce1cc8ec44878c1/pillow-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb", size = 7226887, upload-time = "2026-07-01T11:53:42.764Z" }, + { url = "https://files.pythonhosted.org/packages/fa/17/e324be141d173c1c919428066c3259f21c1b8982e564e01a4a81e96dbdcf/pillow-12.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f", size = 2568039, upload-time = "2026-07-01T11:53:45.372Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c8/0a78b0e02d7ac54bc03e5321c9220da52f0c2ea83b21f7c40e7f3169c502/pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756", size = 5392415, upload-time = "2026-07-01T11:53:47.162Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5b/a02d30018abd97ced9f5a6c63d28597694a00d066516b9c1c6de45859fc9/pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6", size = 4785266, upload-time = "2026-07-01T11:53:49.079Z" }, + { url = "https://files.pythonhosted.org/packages/c8/98/766667a4be768150a202836acd9fad19c06824ca86c4286d3cf6b274964e/pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd", size = 6263814, upload-time = "2026-07-01T11:53:51.32Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2d/ede717bc1144f63886c21fd349bb95860b0d1a21149ff16f2bb362b612b6/pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd", size = 6934408, upload-time = "2026-07-01T11:53:53.487Z" }, + { url = "https://files.pythonhosted.org/packages/a3/48/9c58b685e69d49c31af6c8eb9012055fab7e665785165c84796e2c73ce72/pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c", size = 6337160, upload-time = "2026-07-01T11:53:55.457Z" }, + { url = "https://files.pythonhosted.org/packages/ff/fa/dc2a5c0ba6df93f67c31d34b808b7ce440b40cdbf96f0b81cde1d1e6fa93/pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5", size = 7045172, upload-time = "2026-07-01T11:53:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/86/a5/444817a4d4c4c2417df00513086ca196f388d8f9ef40c2e4ccd1ad1af54b/pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b", size = 6472232, upload-time = "2026-07-01T11:53:59.767Z" }, + { url = "https://files.pythonhosted.org/packages/63/c6/4bad1b18d132a50b27e1365e1ab163616f7a5bb56d330f66f9d1d9d4f9d4/pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a", size = 7233653, upload-time = "2026-07-01T11:54:02.066Z" }, + { url = "https://files.pythonhosted.org/packages/fd/16/00f91ab7760dc842f5aad55217e80fc4a7067a0604535249bc8a2d6d9870/pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26", size = 2568195, upload-time = "2026-07-01T11:54:04.622Z" }, + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, + { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, + { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, + { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, + { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, + { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, + { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, + { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, + { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, + { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, + { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, + { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, + { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/75/18/2e8b40223153ccbc60df07f9e8928dc0c76202aa4e55ae9f53962b6510d6/pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468", size = 5302510, upload-time = "2026-07-01T11:56:25.736Z" }, + { url = "https://files.pythonhosted.org/packages/46/3e/51fabf59d5ab801ceab709453d3ab6b180083496579549de4c45ced6528a/pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94", size = 4736058, upload-time = "2026-07-01T11:56:28.041Z" }, + { url = "https://files.pythonhosted.org/packages/bf/20/22fe9384b7949e25fb1293bcfc84fb82590ff4ea6b37c95b24d26d793d86/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e", size = 5237776, upload-time = "2026-07-01T11:56:30.263Z" }, + { url = "https://files.pythonhosted.org/packages/08/14/f6ba68107680ffa74b39985f3f30884e41318fbc4250caa423c79b4788bb/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3", size = 5860358, upload-time = "2026-07-01T11:56:32.68Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] [[package]] @@ -2850,47 +2473,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "pot" -version = "0.9.6.post1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/8b/5f939eaf1fbeb7ff914fe540d659486951a056e5537b8f454362045b6c72/pot-0.9.6.post1.tar.gz", hash = "sha256:9b6cc14a8daecfe1268268168cf46548f9130976b22b24a9e8ec62a734be6c43", size = 604243, upload-time = "2025-09-22T12:51:14.894Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/65/3ed0362444818585d62521f9bf5e6166b8626a714354bc2c8ea5fbdbcbe6/pot-0.9.6.post1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2127b310a13f03951be450812e7dfdf62c5484bc6219bd0e0639f0347b3b60dd", size = 595401, upload-time = "2025-09-22T12:50:23.421Z" }, - { url = "https://files.pythonhosted.org/packages/07/9b/5145c4264953f03f054d4dc4ce1d8f337eb5827896f9e6a51267432ab86d/pot-0.9.6.post1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7d50dbc851d8b69a6c5305fcad197f149047093e5f4555aed1ea77d1d7823b", size = 464517, upload-time = "2025-09-22T12:50:25.003Z" }, - { url = "https://files.pythonhosted.org/packages/83/23/9724a5a1ebfd4769377d5293208465ef8e803fbcf85350d5d38af349cbea/pot-0.9.6.post1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1de9cf2af8920c5902f1ee779cf2bf388d5677618735ce91f65d7f8e0ead629e", size = 450810, upload-time = "2025-09-22T12:50:26.28Z" }, - { url = "https://files.pythonhosted.org/packages/df/e9/f8f343588d2a18cd0c77fcf6b6f275642dea3cdf4f0e28e16c6e78198aec/pot-0.9.6.post1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b17c1373366f8ebd745d159793f415660ec45e69048305bb8597267d900145ab", size = 1459588, upload-time = "2025-09-22T12:50:27.739Z" }, - { url = "https://files.pythonhosted.org/packages/5b/7d/1529014aebb9d5fd54538115886d005d371a624b1ecaf5c2525b45ad0f77/pot-0.9.6.post1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48924f34d61b909e68651f3fe9fc1a892c69ae38d3c52bc832f95a28569c0e0e", size = 1478099, upload-time = "2025-09-22T12:50:29.201Z" }, - { url = "https://files.pythonhosted.org/packages/4e/87/84cfc49d4d0eb3e7b6cfc8352f0e73f62d456f6ce875da612b919a6bff6f/pot-0.9.6.post1-cp310-cp310-win32.whl", hash = "sha256:06e21b4dcebc2e8e318a96889243580ea64364830d05d53c4d038afedbe072cc", size = 443775, upload-time = "2025-09-22T12:50:30.84Z" }, - { url = "https://files.pythonhosted.org/packages/c4/21/9731ac0b125f755bb513a4ee081dca0ca5335e9059fb3332dd7c50d28415/pot-0.9.6.post1-cp310-cp310-win_amd64.whl", hash = "sha256:d35bb0169ef242fc2ce4f610572a5d11ac11d646698cbdf8cbb45d828f3c514b", size = 458481, upload-time = "2025-09-22T12:50:32.431Z" }, - { url = "https://files.pythonhosted.org/packages/f6/fc/3f4014bd6713c5b4c8a329b12c52842443b2284f52213a80e697b76b9f20/pot-0.9.6.post1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7fd8482a0262e5c875c05cf52e9c087e7c8bc473ef05d175887ad16e3c0443b7", size = 599499, upload-time = "2025-09-22T12:50:33.796Z" }, - { url = "https://files.pythonhosted.org/packages/e7/4e/b22b789ee3a81c11c6f39ff08ed6a2e797a2a75a831fae996f4057db4771/pot-0.9.6.post1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c0bfac9daec0095061279a709f52be740e09363a62fe4c7edc843a4a0f6144c6", size = 466484, upload-time = "2025-09-22T12:50:34.973Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ae/2b35b96562bd72baf6de9583458878738f4508eef70d6fa9dd5867760d6a/pot-0.9.6.post1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:703853f7ba0ae2afed8203ea3478e87ef5f39d55cd75b1a39bb622867d1d5628", size = 453014, upload-time = "2025-09-22T12:50:36.157Z" }, - { url = "https://files.pythonhosted.org/packages/44/7e/f49d0593338a3b7f2c88c4cd6f1285c084e8ce05d52d42ac6f89f4f7ec0c/pot-0.9.6.post1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68268b4dd926976cf0604d466a57dff2ca44372e8ae9c879ba1f3d2a51e3be3d", size = 1494875, upload-time = "2025-09-22T12:50:37.903Z" }, - { url = "https://files.pythonhosted.org/packages/15/91/844c8437caaca6d6a71b38623df75c43642a116d399316adb1d0a9280c85/pot-0.9.6.post1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7568ddc957d3a16739bd24f9e07ce655166d27ebbc8786aad692cc5ba5d4c59", size = 1514551, upload-time = "2025-09-22T12:50:39.616Z" }, - { url = "https://files.pythonhosted.org/packages/ac/de/34a50565c37c0b71725a8075ff1ad2de62213d2e119276b546ef20356ac2/pot-0.9.6.post1-cp311-cp311-win32.whl", hash = "sha256:9649b736ea5dddad3a89d55a4a3bb0078610307ba64cac2efebe6bfcf8cfe785", size = 443490, upload-time = "2025-09-22T12:50:41.162Z" }, - { url = "https://files.pythonhosted.org/packages/a7/fa/453730c1b10094ab4d2ecd0b5fbfcdfe0305419cf01e32a2d31efd333559/pot-0.9.6.post1-cp311-cp311-win_amd64.whl", hash = "sha256:e161e49a22d5a925993baace4679f4e32fc2ade8f45ad73cf8417e13df5bd337", size = 458509, upload-time = "2025-09-22T12:50:43.597Z" }, - { url = "https://files.pythonhosted.org/packages/b9/28/13622807461f9f6082a8cd6768f9b4a810bc3a8fda474b81572da94b4d23/pot-0.9.6.post1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f7c542fc20662e35c24dd82eeff8a737220757434d7f0038664a7322221452f7", size = 599240, upload-time = "2025-09-22T12:50:44.848Z" }, - { url = "https://files.pythonhosted.org/packages/c6/5c/b4e017560531f53d06798c681b0d0a9488bb8116bc98da9d399a3d096391/pot-0.9.6.post1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c1755516a7354cbd6110ad2e5f341b98b9968240c2f0f67b0ff5e3ebcb3105bd", size = 464695, upload-time = "2025-09-22T12:50:46.341Z" }, - { url = "https://files.pythonhosted.org/packages/07/9f/57e49b3f7173359741053c5e2766a45dcf649d767c2e967ef93526c9045f/pot-0.9.6.post1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3207362d3e3b5aaa783f452aa85f66e83edbefb5764f34662860af54ac72ee6", size = 454726, upload-time = "2025-09-22T12:50:47.953Z" }, - { url = "https://files.pythonhosted.org/packages/30/60/fa72dd6094f7dbe6b38e2c6907af8cd0f18c6bd107e0cf4874deddaba883/pot-0.9.6.post1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f6659c5657e6d7e9f98f4a82e0ed64f88e9fce69b2e557416d156343919ba3", size = 1503391, upload-time = "2025-09-22T12:50:49.336Z" }, - { url = "https://files.pythonhosted.org/packages/2f/3f/cc519c1176116271b6282268a705162fa042c16cc922bc56039445c9d697/pot-0.9.6.post1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f1b0148ae17bec0ed12264c6da3a05e13913b716e2a8c9043242b5d8349d8df", size = 1528170, upload-time = "2025-09-22T12:50:50.625Z" }, - { url = "https://files.pythonhosted.org/packages/f5/01/0132c94404cd0b1b2f21c4a49698db9dcd6107c47c02b22df1ed38206b2a/pot-0.9.6.post1-cp312-cp312-win32.whl", hash = "sha256:571e543cc2b0a462365002203595baf2b89c3d064cce4fce70fd1231e832c21f", size = 440577, upload-time = "2025-09-22T12:50:51.716Z" }, - { url = "https://files.pythonhosted.org/packages/c1/6d/23229c0e198a4f7fb27750b3ef8497e6ebed23fe531ed64b5194da8b2b02/pot-0.9.6.post1-cp312-cp312-win_amd64.whl", hash = "sha256:b1d8bd9a334c72baa37f9a2b268de5366c23c0f9c9e3d6dc25d150137ec2823c", size = 455404, upload-time = "2025-09-22T12:50:52.956Z" }, - { url = "https://files.pythonhosted.org/packages/53/17/e4aebb8deef58b0d40ac339d952d12c63559801b50ae43c622d49bebda7e/pot-0.9.6.post1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:659fff750a162f58b52b33a64c4ac358f4ff44e9dff0841052c088e1b6a54430", size = 596485, upload-time = "2025-09-22T12:50:54.309Z" }, - { url = "https://files.pythonhosted.org/packages/f7/b9/3646c153b13f999ac30112dcf85c5f233af79b0d98c37b52dda9a624c91b/pot-0.9.6.post1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4f54830e9f9cb78b1ff7abd5c5bf162625ed6aea903241267c64ea9f0fb73ddb", size = 463244, upload-time = "2025-09-22T12:50:56.004Z" }, - { url = "https://files.pythonhosted.org/packages/53/e9/c7092f7aec8cb32739ad66ba1f1259626546e4893b61b905ce2da3987235/pot-0.9.6.post1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e9fd4b1fafacd37debdb984687ddb26f5c43d1429401847d388a6f1bd1f10e98", size = 453215, upload-time = "2025-09-22T12:50:57.515Z" }, - { url = "https://files.pythonhosted.org/packages/0c/a1/f0187ab15aa1538ece07b28f3a7938b8592ef01fbe37b1a8f9c2f8f47f4d/pot-0.9.6.post1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ec097ec0ef8bb93fee8cdb187b6a0a9653613cba7b06bb603247930e2c629cdc", size = 1496245, upload-time = "2025-09-22T12:50:58.848Z" }, - { url = "https://files.pythonhosted.org/packages/29/fa/85af71553b7e990fc37da8d5f2e7294ec66297e62cba419efeec11518e5a/pot-0.9.6.post1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:299f11f172908d799793ef18b2bc82452305350d2528d243e255a17876e98a57", size = 1521691, upload-time = "2025-09-22T12:51:00.203Z" }, - { url = "https://files.pythonhosted.org/packages/19/ae/96b2bce173b3d2d3d0faf8b7362fe79e60e1a6a939c9459b2f7b64e625d8/pot-0.9.6.post1-cp313-cp313-win32.whl", hash = "sha256:8a1d95310faae9c75355d9e2fac8dfac41316a2450061eefc982ee498a687a34", size = 439760, upload-time = "2025-09-22T12:51:01.601Z" }, - { url = "https://files.pythonhosted.org/packages/f7/b1/8ca34418e7c4a2ec666e2204539577287223c4e78ab80b1c746cedb559c3/pot-0.9.6.post1-cp313-cp313-win_amd64.whl", hash = "sha256:a43e2b61389bd32f5b488da2488999ed55867e95fedb25dd64f9f390e40b4fab", size = 454228, upload-time = "2025-09-22T12:51:03.215Z" }, -] - [[package]] name = "pre-commit" version = "4.6.0" @@ -3194,24 +2776,6 @@ crypto = [ { name = "cryptography" }, ] -[[package]] -name = "pynndescent" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "joblib", marker = "python_full_version < '3.13'" }, - { name = "llvmlite", marker = "python_full_version < '3.13'" }, - { name = "numba", marker = "python_full_version < '3.13'" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4a/fb/7f58c397fb31666756457ee2ac4c0289ef2daad57f4ae4be8dec12f80b03/pynndescent-0.6.0.tar.gz", hash = "sha256:7ffde0fb5b400741e055a9f7d377e3702e02250616834231f6c209e39aac24f5", size = 2992987, upload-time = "2026-01-08T21:29:58.943Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/e6/94145d714402fd5ade00b5661f2d0ab981219e07f7db9bfa16786cdb9c04/pynndescent-0.6.0-py3-none-any.whl", hash = "sha256:dc8c74844e4c7f5cbd1e0cd6909da86fdc789e6ff4997336e344779c3d5538ef", size = 73511, upload-time = "2026-01-08T21:29:57.306Z" }, -] - [[package]] name = "pyparsing" version = "3.3.2" @@ -3862,265 +3426,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/72/c6c32d2b657fa3dad1de340254e14390b1e334ce38268b7ad51abda3c8c2/s3transfer-0.17.0-py3-none-any.whl", hash = "sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20", size = 86811, upload-time = "2026-04-29T22:07:34.966Z" }, ] -[[package]] -name = "scikit-learn" -version = "1.7.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "joblib", marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/3e/daed796fd69cce768b8788401cc464ea90b306fb196ae1ffed0b98182859/scikit_learn-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b33579c10a3081d076ab403df4a4190da4f4432d443521674637677dc91e61f", size = 9336221, upload-time = "2025-09-09T08:20:19.328Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ce/af9d99533b24c55ff4e18d9b7b4d9919bbc6cd8f22fe7a7be01519a347d5/scikit_learn-1.7.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:36749fb62b3d961b1ce4fedf08fa57a1986cd409eff2d783bca5d4b9b5fce51c", size = 8653834, upload-time = "2025-09-09T08:20:22.073Z" }, - { url = "https://files.pythonhosted.org/packages/58/0e/8c2a03d518fb6bd0b6b0d4b114c63d5f1db01ff0f9925d8eb10960d01c01/scikit_learn-1.7.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7a58814265dfc52b3295b1900cfb5701589d30a8bb026c7540f1e9d3499d5ec8", size = 9660938, upload-time = "2025-09-09T08:20:24.327Z" }, - { url = "https://files.pythonhosted.org/packages/2b/75/4311605069b5d220e7cf5adabb38535bd96f0079313cdbb04b291479b22a/scikit_learn-1.7.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a847fea807e278f821a0406ca01e387f97653e284ecbd9750e3ee7c90347f18", size = 9477818, upload-time = "2025-09-09T08:20:26.845Z" }, - { url = "https://files.pythonhosted.org/packages/7f/9b/87961813c34adbca21a6b3f6b2bea344c43b30217a6d24cc437c6147f3e8/scikit_learn-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:ca250e6836d10e6f402436d6463d6c0e4d8e0234cfb6a9a47835bd392b852ce5", size = 8886969, upload-time = "2025-09-09T08:20:29.329Z" }, - { url = "https://files.pythonhosted.org/packages/43/83/564e141eef908a5863a54da8ca342a137f45a0bfb71d1d79704c9894c9d1/scikit_learn-1.7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7509693451651cd7361d30ce4e86a1347493554f172b1c72a39300fa2aea79e", size = 9331967, upload-time = "2025-09-09T08:20:32.421Z" }, - { url = "https://files.pythonhosted.org/packages/18/d6/ba863a4171ac9d7314c4d3fc251f015704a2caeee41ced89f321c049ed83/scikit_learn-1.7.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:0486c8f827c2e7b64837c731c8feff72c0bd2b998067a8a9cbc10643c31f0fe1", size = 8648645, upload-time = "2025-09-09T08:20:34.436Z" }, - { url = "https://files.pythonhosted.org/packages/ef/0e/97dbca66347b8cf0ea8b529e6bb9367e337ba2e8be0ef5c1a545232abfde/scikit_learn-1.7.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:89877e19a80c7b11a2891a27c21c4894fb18e2c2e077815bcade10d34287b20d", size = 9715424, upload-time = "2025-09-09T08:20:36.776Z" }, - { url = "https://files.pythonhosted.org/packages/f7/32/1f3b22e3207e1d2c883a7e09abb956362e7d1bd2f14458c7de258a26ac15/scikit_learn-1.7.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8da8bf89d4d79aaec192d2bda62f9b56ae4e5b4ef93b6a56b5de4977e375c1f1", size = 9509234, upload-time = "2025-09-09T08:20:38.957Z" }, - { url = "https://files.pythonhosted.org/packages/9f/71/34ddbd21f1da67c7a768146968b4d0220ee6831e4bcbad3e03dd3eae88b6/scikit_learn-1.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:9b7ed8d58725030568523e937c43e56bc01cadb478fc43c042a9aca1dacb3ba1", size = 8894244, upload-time = "2025-09-09T08:20:41.166Z" }, - { url = "https://files.pythonhosted.org/packages/a7/aa/3996e2196075689afb9fce0410ebdb4a09099d7964d061d7213700204409/scikit_learn-1.7.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8d91a97fa2b706943822398ab943cde71858a50245e31bc71dba62aab1d60a96", size = 9259818, upload-time = "2025-09-09T08:20:43.19Z" }, - { url = "https://files.pythonhosted.org/packages/43/5d/779320063e88af9c4a7c2cf463ff11c21ac9c8bd730c4a294b0000b666c9/scikit_learn-1.7.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:acbc0f5fd2edd3432a22c69bed78e837c70cf896cd7993d71d51ba6708507476", size = 8636997, upload-time = "2025-09-09T08:20:45.468Z" }, - { url = "https://files.pythonhosted.org/packages/5c/d0/0c577d9325b05594fdd33aa970bf53fb673f051a45496842caee13cfd7fe/scikit_learn-1.7.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e5bf3d930aee75a65478df91ac1225ff89cd28e9ac7bd1196853a9229b6adb0b", size = 9478381, upload-time = "2025-09-09T08:20:47.982Z" }, - { url = "https://files.pythonhosted.org/packages/82/70/8bf44b933837ba8494ca0fc9a9ab60f1c13b062ad0197f60a56e2fc4c43e/scikit_learn-1.7.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4d6e9deed1a47aca9fe2f267ab8e8fe82ee20b4526b2c0cd9e135cea10feb44", size = 9300296, upload-time = "2025-09-09T08:20:50.366Z" }, - { url = "https://files.pythonhosted.org/packages/c6/99/ed35197a158f1fdc2fe7c3680e9c70d0128f662e1fee4ed495f4b5e13db0/scikit_learn-1.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:6088aa475f0785e01bcf8529f55280a3d7d298679f50c0bb70a2364a82d0b290", size = 8731256, upload-time = "2025-09-09T08:20:52.627Z" }, - { url = "https://files.pythonhosted.org/packages/ae/93/a3038cb0293037fd335f77f31fe053b89c72f17b1c8908c576c29d953e84/scikit_learn-1.7.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0b7dacaa05e5d76759fb071558a8b5130f4845166d88654a0f9bdf3eb57851b7", size = 9212382, upload-time = "2025-09-09T08:20:54.731Z" }, - { url = "https://files.pythonhosted.org/packages/40/dd/9a88879b0c1104259136146e4742026b52df8540c39fec21a6383f8292c7/scikit_learn-1.7.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:abebbd61ad9e1deed54cca45caea8ad5f79e1b93173dece40bb8e0c658dbe6fe", size = 8592042, upload-time = "2025-09-09T08:20:57.313Z" }, - { url = "https://files.pythonhosted.org/packages/46/af/c5e286471b7d10871b811b72ae794ac5fe2989c0a2df07f0ec723030f5f5/scikit_learn-1.7.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:502c18e39849c0ea1a5d681af1dbcf15f6cce601aebb657aabbfe84133c1907f", size = 9434180, upload-time = "2025-09-09T08:20:59.671Z" }, - { url = "https://files.pythonhosted.org/packages/f1/fd/df59faa53312d585023b2da27e866524ffb8faf87a68516c23896c718320/scikit_learn-1.7.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a4c328a71785382fe3fe676a9ecf2c86189249beff90bf85e22bdb7efaf9ae0", size = 9283660, upload-time = "2025-09-09T08:21:01.71Z" }, - { url = "https://files.pythonhosted.org/packages/a7/c7/03000262759d7b6f38c836ff9d512f438a70d8a8ddae68ee80de72dcfb63/scikit_learn-1.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:63a9afd6f7b229aad94618c01c252ce9e6fa97918c5ca19c9a17a087d819440c", size = 8702057, upload-time = "2025-09-09T08:21:04.234Z" }, - { url = "https://files.pythonhosted.org/packages/55/87/ef5eb1f267084532c8e4aef98a28b6ffe7425acbfd64b5e2f2e066bc29b3/scikit_learn-1.7.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9acb6c5e867447b4e1390930e3944a005e2cb115922e693c08a323421a6966e8", size = 9558731, upload-time = "2025-09-09T08:21:06.381Z" }, - { url = "https://files.pythonhosted.org/packages/93/f8/6c1e3fc14b10118068d7938878a9f3f4e6d7b74a8ddb1e5bed65159ccda8/scikit_learn-1.7.2-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:2a41e2a0ef45063e654152ec9d8bcfc39f7afce35b08902bfe290c2498a67a6a", size = 9038852, upload-time = "2025-09-09T08:21:08.628Z" }, - { url = "https://files.pythonhosted.org/packages/83/87/066cafc896ee540c34becf95d30375fe5cbe93c3b75a0ee9aa852cd60021/scikit_learn-1.7.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98335fb98509b73385b3ab2bd0639b1f610541d3988ee675c670371d6a87aa7c", size = 9527094, upload-time = "2025-09-09T08:21:11.486Z" }, - { url = "https://files.pythonhosted.org/packages/9c/2b/4903e1ccafa1f6453b1ab78413938c8800633988c838aa0be386cbb33072/scikit_learn-1.7.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191e5550980d45449126e23ed1d5e9e24b2c68329ee1f691a3987476e115e09c", size = 9367436, upload-time = "2025-09-09T08:21:13.602Z" }, - { url = "https://files.pythonhosted.org/packages/b5/aa/8444be3cfb10451617ff9d177b3c190288f4563e6c50ff02728be67ad094/scikit_learn-1.7.2-cp313-cp313t-win_amd64.whl", hash = "sha256:57dc4deb1d3762c75d685507fbd0bc17160144b2f2ba4ccea5dc285ab0d0e973", size = 9275749, upload-time = "2025-09-09T08:21:15.96Z" }, - { url = "https://files.pythonhosted.org/packages/d9/82/dee5acf66837852e8e68df6d8d3a6cb22d3df997b733b032f513d95205b7/scikit_learn-1.7.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fa8f63940e29c82d1e67a45d5297bdebbcb585f5a5a50c4914cc2e852ab77f33", size = 9208906, upload-time = "2025-09-09T08:21:18.557Z" }, - { url = "https://files.pythonhosted.org/packages/3c/30/9029e54e17b87cb7d50d51a5926429c683d5b4c1732f0507a6c3bed9bf65/scikit_learn-1.7.2-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:f95dc55b7902b91331fa4e5845dd5bde0580c9cd9612b1b2791b7e80c3d32615", size = 8627836, upload-time = "2025-09-09T08:21:20.695Z" }, - { url = "https://files.pythonhosted.org/packages/60/18/4a52c635c71b536879f4b971c2cedf32c35ee78f48367885ed8025d1f7ee/scikit_learn-1.7.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9656e4a53e54578ad10a434dc1f993330568cfee176dff07112b8785fb413106", size = 9426236, upload-time = "2025-09-09T08:21:22.645Z" }, - { url = "https://files.pythonhosted.org/packages/99/7e/290362f6ab582128c53445458a5befd471ed1ea37953d5bcf80604619250/scikit_learn-1.7.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96dc05a854add0e50d3f47a1ef21a10a595016da5b007c7d9cd9d0bffd1fcc61", size = 9312593, upload-time = "2025-09-09T08:21:24.65Z" }, - { url = "https://files.pythonhosted.org/packages/8e/87/24f541b6d62b1794939ae6422f8023703bbf6900378b2b34e0b4384dfefd/scikit_learn-1.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:bb24510ed3f9f61476181e4db51ce801e2ba37541def12dc9333b946fc7a9cf8", size = 8820007, upload-time = "2025-09-09T08:21:26.713Z" }, -] - -[[package]] -name = "scikit-learn" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/92/53ea2181da8ac6bf27170191028aee7251f8f841f8d3edbfdcaf2008fde9/scikit_learn-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:146b4d36f800c013d267b29168813f7a03a43ecd2895d04861f1240b564421da", size = 8595835, upload-time = "2025-12-10T07:07:39.385Z" }, - { url = "https://files.pythonhosted.org/packages/01/18/d154dc1638803adf987910cdd07097d9c526663a55666a97c124d09fb96a/scikit_learn-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f984ca4b14914e6b4094c5d52a32ea16b49832c03bd17a110f004db3c223e8e1", size = 8080381, upload-time = "2025-12-10T07:07:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/8a/44/226142fcb7b7101e64fdee5f49dbe6288d4c7af8abf593237b70fca080a4/scikit_learn-1.8.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e30adb87f0cc81c7690a84f7932dd66be5bac57cfe16b91cb9151683a4a2d3b", size = 8799632, upload-time = "2025-12-10T07:07:43.899Z" }, - { url = "https://files.pythonhosted.org/packages/36/4d/4a67f30778a45d542bbea5db2dbfa1e9e100bf9ba64aefe34215ba9f11f6/scikit_learn-1.8.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ada8121bcb4dac28d930febc791a69f7cb1673c8495e5eee274190b73a4559c1", size = 9103788, upload-time = "2025-12-10T07:07:45.982Z" }, - { url = "https://files.pythonhosted.org/packages/89/3c/45c352094cfa60050bcbb967b1faf246b22e93cb459f2f907b600f2ceda5/scikit_learn-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:c57b1b610bd1f40ba43970e11ce62821c2e6569e4d74023db19c6b26f246cb3b", size = 8081706, upload-time = "2025-12-10T07:07:48.111Z" }, - { url = "https://files.pythonhosted.org/packages/3d/46/5416595bb395757f754feb20c3d776553a386b661658fb21b7c814e89efe/scikit_learn-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:2838551e011a64e3053ad7618dda9310175f7515f1742fa2d756f7c874c05961", size = 7688451, upload-time = "2025-12-10T07:07:49.873Z" }, - { url = "https://files.pythonhosted.org/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" }, - { url = "https://files.pythonhosted.org/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" }, - { url = "https://files.pythonhosted.org/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" }, - { url = "https://files.pythonhosted.org/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" }, - { url = "https://files.pythonhosted.org/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" }, - { url = "https://files.pythonhosted.org/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" }, - { url = "https://files.pythonhosted.org/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" }, - { url = "https://files.pythonhosted.org/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" }, - { url = "https://files.pythonhosted.org/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" }, - { url = "https://files.pythonhosted.org/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" }, - { url = "https://files.pythonhosted.org/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" }, - { url = "https://files.pythonhosted.org/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" }, - { url = "https://files.pythonhosted.org/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" }, - { url = "https://files.pythonhosted.org/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" }, - { url = "https://files.pythonhosted.org/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" }, - { url = "https://files.pythonhosted.org/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" }, - { url = "https://files.pythonhosted.org/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" }, - { url = "https://files.pythonhosted.org/packages/24/05/1af2c186174cc92dcab2233f327336058c077d38f6fe2aceb08e6ab4d509/scikit_learn-1.8.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c22a2da7a198c28dd1a6e1136f19c830beab7fdca5b3e5c8bba8394f8a5c45b3", size = 8528667, upload-time = "2025-12-10T07:08:27.541Z" }, - { url = "https://files.pythonhosted.org/packages/a8/25/01c0af38fe969473fb292bba9dc2b8f9b451f3112ff242c647fee3d0dfe7/scikit_learn-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:6b595b07a03069a2b1740dc08c2299993850ea81cce4fe19b2421e0c970de6b7", size = 8066524, upload-time = "2025-12-10T07:08:29.822Z" }, - { url = "https://files.pythonhosted.org/packages/be/ce/a0623350aa0b68647333940ee46fe45086c6060ec604874e38e9ab7d8e6c/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:29ffc74089f3d5e87dfca4c2c8450f88bdc61b0fc6ed5d267f3988f19a1309f6", size = 8657133, upload-time = "2025-12-10T07:08:31.865Z" }, - { url = "https://files.pythonhosted.org/packages/b8/cb/861b41341d6f1245e6ca80b1c1a8c4dfce43255b03df034429089ca2a2c5/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb65db5d7531bccf3a4f6bec3462223bea71384e2cda41da0f10b7c292b9e7c4", size = 8923223, upload-time = "2025-12-10T07:08:34.166Z" }, - { url = "https://files.pythonhosted.org/packages/76/18/a8def8f91b18cd1ba6e05dbe02540168cb24d47e8dcf69e8d00b7da42a08/scikit_learn-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:56079a99c20d230e873ea40753102102734c5953366972a71d5cb39a32bc40c6", size = 8096518, upload-time = "2025-12-10T07:08:36.339Z" }, - { url = "https://files.pythonhosted.org/packages/d1/77/482076a678458307f0deb44e29891d6022617b2a64c840c725495bee343f/scikit_learn-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:3bad7565bc9cf37ce19a7c0d107742b320c1285df7aab1a6e2d28780df167242", size = 7754546, upload-time = "2025-12-10T07:08:38.128Z" }, - { url = "https://files.pythonhosted.org/packages/2d/d1/ef294ca754826daa043b2a104e59960abfab4cf653891037d19dd5b6f3cf/scikit_learn-1.8.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4511be56637e46c25721e83d1a9cea9614e7badc7040c4d573d75fbe257d6fd7", size = 8848305, upload-time = "2025-12-10T07:08:41.013Z" }, - { url = "https://files.pythonhosted.org/packages/5b/e2/b1f8b05138ee813b8e1a4149f2f0d289547e60851fd1bb268886915adbda/scikit_learn-1.8.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:a69525355a641bf8ef136a7fa447672fb54fe8d60cab5538d9eb7c6438543fb9", size = 8432257, upload-time = "2025-12-10T07:08:42.873Z" }, - { url = "https://files.pythonhosted.org/packages/26/11/c32b2138a85dcb0c99f6afd13a70a951bfdff8a6ab42d8160522542fb647/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2656924ec73e5939c76ac4c8b026fc203b83d8900362eb2599d8aee80e4880f", size = 8678673, upload-time = "2025-12-10T07:08:45.362Z" }, - { url = "https://files.pythonhosted.org/packages/c7/57/51f2384575bdec454f4fe4e7a919d696c9ebce914590abf3e52d47607ab8/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15fc3b5d19cc2be65404786857f2e13c70c83dd4782676dd6814e3b89dc8f5b9", size = 8922467, upload-time = "2025-12-10T07:08:47.408Z" }, - { url = "https://files.pythonhosted.org/packages/35/4d/748c9e2872637a57981a04adc038dacaa16ba8ca887b23e34953f0b3f742/scikit_learn-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:00d6f1d66fbcf4eba6e356e1420d33cc06c70a45bb1363cd6f6a8e4ebbbdece2", size = 8774395, upload-time = "2025-12-10T07:08:49.337Z" }, - { url = "https://files.pythonhosted.org/packages/60/22/d7b2ebe4704a5e50790ba089d5c2ae308ab6bb852719e6c3bd4f04c3a363/scikit_learn-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f28dd15c6bb0b66ba09728cf09fd8736c304be29409bd8445a080c1280619e8c", size = 8002647, upload-time = "2025-12-10T07:08:51.601Z" }, -] - -[[package]] -name = "scipy" -version = "1.15.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/2f/4966032c5f8cc7e6a60f1b2e0ad686293b9474b65246b0c642e3ef3badd0/scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c", size = 38702770, upload-time = "2025-05-08T16:04:20.849Z" }, - { url = "https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253", size = 30094511, upload-time = "2025-05-08T16:04:27.103Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b1/4deb37252311c1acff7f101f6453f0440794f51b6eacb1aad4459a134081/scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f", size = 22368151, upload-time = "2025-05-08T16:04:31.731Z" }, - { url = "https://files.pythonhosted.org/packages/38/7d/f457626e3cd3c29b3a49ca115a304cebb8cc6f31b04678f03b216899d3c6/scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92", size = 25121732, upload-time = "2025-05-08T16:04:36.596Z" }, - { url = "https://files.pythonhosted.org/packages/db/0a/92b1de4a7adc7a15dcf5bddc6e191f6f29ee663b30511ce20467ef9b82e4/scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82", size = 35547617, upload-time = "2025-05-08T16:04:43.546Z" }, - { url = "https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40", size = 37662964, upload-time = "2025-05-08T16:04:49.431Z" }, - { url = "https://files.pythonhosted.org/packages/25/e1/3df8f83cb15f3500478c889be8fb18700813b95e9e087328230b98d547ff/scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e", size = 37238749, upload-time = "2025-05-08T16:04:55.215Z" }, - { url = "https://files.pythonhosted.org/packages/93/3e/b3257cf446f2a3533ed7809757039016b74cd6f38271de91682aa844cfc5/scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c", size = 40022383, upload-time = "2025-05-08T16:05:01.914Z" }, - { url = "https://files.pythonhosted.org/packages/d1/84/55bc4881973d3f79b479a5a2e2df61c8c9a04fcb986a213ac9c02cfb659b/scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13", size = 41259201, upload-time = "2025-05-08T16:05:08.166Z" }, - { url = "https://files.pythonhosted.org/packages/96/ab/5cc9f80f28f6a7dff646c5756e559823614a42b1939d86dd0ed550470210/scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b", size = 38714255, upload-time = "2025-05-08T16:05:14.596Z" }, - { url = "https://files.pythonhosted.org/packages/4a/4a/66ba30abe5ad1a3ad15bfb0b59d22174012e8056ff448cb1644deccbfed2/scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba", size = 30111035, upload-time = "2025-05-08T16:05:20.152Z" }, - { url = "https://files.pythonhosted.org/packages/4b/fa/a7e5b95afd80d24313307f03624acc65801846fa75599034f8ceb9e2cbf6/scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65", size = 22384499, upload-time = "2025-05-08T16:05:24.494Z" }, - { url = "https://files.pythonhosted.org/packages/17/99/f3aaddccf3588bb4aea70ba35328c204cadd89517a1612ecfda5b2dd9d7a/scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1", size = 25152602, upload-time = "2025-05-08T16:05:29.313Z" }, - { url = "https://files.pythonhosted.org/packages/56/c5/1032cdb565f146109212153339f9cb8b993701e9fe56b1c97699eee12586/scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889", size = 35503415, upload-time = "2025-05-08T16:05:34.699Z" }, - { url = "https://files.pythonhosted.org/packages/bd/37/89f19c8c05505d0601ed5650156e50eb881ae3918786c8fd7262b4ee66d3/scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982", size = 37652622, upload-time = "2025-05-08T16:05:40.762Z" }, - { url = "https://files.pythonhosted.org/packages/7e/31/be59513aa9695519b18e1851bb9e487de66f2d31f835201f1b42f5d4d475/scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9", size = 37244796, upload-time = "2025-05-08T16:05:48.119Z" }, - { url = "https://files.pythonhosted.org/packages/10/c0/4f5f3eeccc235632aab79b27a74a9130c6c35df358129f7ac8b29f562ac7/scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594", size = 40047684, upload-time = "2025-05-08T16:05:54.22Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a7/0ddaf514ce8a8714f6ed243a2b391b41dbb65251affe21ee3077ec45ea9a/scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb", size = 41246504, upload-time = "2025-05-08T16:06:00.437Z" }, - { url = "https://files.pythonhosted.org/packages/37/4b/683aa044c4162e10ed7a7ea30527f2cbd92e6999c10a8ed8edb253836e9c/scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019", size = 38766735, upload-time = "2025-05-08T16:06:06.471Z" }, - { url = "https://files.pythonhosted.org/packages/7b/7e/f30be3d03de07f25dc0ec926d1681fed5c732d759ac8f51079708c79e680/scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6", size = 30173284, upload-time = "2025-05-08T16:06:11.686Z" }, - { url = "https://files.pythonhosted.org/packages/07/9c/0ddb0d0abdabe0d181c1793db51f02cd59e4901da6f9f7848e1f96759f0d/scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477", size = 22446958, upload-time = "2025-05-08T16:06:15.97Z" }, - { url = "https://files.pythonhosted.org/packages/af/43/0bce905a965f36c58ff80d8bea33f1f9351b05fad4beaad4eae34699b7a1/scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c", size = 25242454, upload-time = "2025-05-08T16:06:20.394Z" }, - { url = "https://files.pythonhosted.org/packages/56/30/a6f08f84ee5b7b28b4c597aca4cbe545535c39fe911845a96414700b64ba/scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45", size = 35210199, upload-time = "2025-05-08T16:06:26.159Z" }, - { url = "https://files.pythonhosted.org/packages/0b/1f/03f52c282437a168ee2c7c14a1a0d0781a9a4a8962d84ac05c06b4c5b555/scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49", size = 37309455, upload-time = "2025-05-08T16:06:32.778Z" }, - { url = "https://files.pythonhosted.org/packages/89/b1/fbb53137f42c4bf630b1ffdfc2151a62d1d1b903b249f030d2b1c0280af8/scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e", size = 36885140, upload-time = "2025-05-08T16:06:39.249Z" }, - { url = "https://files.pythonhosted.org/packages/2e/2e/025e39e339f5090df1ff266d021892694dbb7e63568edcfe43f892fa381d/scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539", size = 39710549, upload-time = "2025-05-08T16:06:45.729Z" }, - { url = "https://files.pythonhosted.org/packages/e6/eb/3bf6ea8ab7f1503dca3a10df2e4b9c3f6b3316df07f6c0ded94b281c7101/scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed", size = 40966184, upload-time = "2025-05-08T16:06:52.623Z" }, - { url = "https://files.pythonhosted.org/packages/73/18/ec27848c9baae6e0d6573eda6e01a602e5649ee72c27c3a8aad673ebecfd/scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759", size = 38728256, upload-time = "2025-05-08T16:06:58.696Z" }, - { url = "https://files.pythonhosted.org/packages/74/cd/1aef2184948728b4b6e21267d53b3339762c285a46a274ebb7863c9e4742/scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62", size = 30109540, upload-time = "2025-05-08T16:07:04.209Z" }, - { url = "https://files.pythonhosted.org/packages/5b/d8/59e452c0a255ec352bd0a833537a3bc1bfb679944c4938ab375b0a6b3a3e/scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb", size = 22383115, upload-time = "2025-05-08T16:07:08.998Z" }, - { url = "https://files.pythonhosted.org/packages/08/f5/456f56bbbfccf696263b47095291040655e3cbaf05d063bdc7c7517f32ac/scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730", size = 25163884, upload-time = "2025-05-08T16:07:14.091Z" }, - { url = "https://files.pythonhosted.org/packages/a2/66/a9618b6a435a0f0c0b8a6d0a2efb32d4ec5a85f023c2b79d39512040355b/scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825", size = 35174018, upload-time = "2025-05-08T16:07:19.427Z" }, - { url = "https://files.pythonhosted.org/packages/b5/09/c5b6734a50ad4882432b6bb7c02baf757f5b2f256041da5df242e2d7e6b6/scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7", size = 37269716, upload-time = "2025-05-08T16:07:25.712Z" }, - { url = "https://files.pythonhosted.org/packages/77/0a/eac00ff741f23bcabd352731ed9b8995a0a60ef57f5fd788d611d43d69a1/scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11", size = 36872342, upload-time = "2025-05-08T16:07:31.468Z" }, - { url = "https://files.pythonhosted.org/packages/fe/54/4379be86dd74b6ad81551689107360d9a3e18f24d20767a2d5b9253a3f0a/scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126", size = 39670869, upload-time = "2025-05-08T16:07:38.002Z" }, - { url = "https://files.pythonhosted.org/packages/87/2e/892ad2862ba54f084ffe8cc4a22667eaf9c2bcec6d2bff1d15713c6c0703/scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163", size = 40988851, upload-time = "2025-05-08T16:08:33.671Z" }, - { url = "https://files.pythonhosted.org/packages/1b/e9/7a879c137f7e55b30d75d90ce3eb468197646bc7b443ac036ae3fe109055/scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8", size = 38863011, upload-time = "2025-05-08T16:07:44.039Z" }, - { url = "https://files.pythonhosted.org/packages/51/d1/226a806bbd69f62ce5ef5f3ffadc35286e9fbc802f606a07eb83bf2359de/scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5", size = 30266407, upload-time = "2025-05-08T16:07:49.891Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9b/f32d1d6093ab9eeabbd839b0f7619c62e46cc4b7b6dbf05b6e615bbd4400/scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e", size = 22540030, upload-time = "2025-05-08T16:07:54.121Z" }, - { url = "https://files.pythonhosted.org/packages/e7/29/c278f699b095c1a884f29fda126340fcc201461ee8bfea5c8bdb1c7c958b/scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb", size = 25218709, upload-time = "2025-05-08T16:07:58.506Z" }, - { url = "https://files.pythonhosted.org/packages/24/18/9e5374b617aba742a990581373cd6b68a2945d65cc588482749ef2e64467/scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723", size = 34809045, upload-time = "2025-05-08T16:08:03.929Z" }, - { url = "https://files.pythonhosted.org/packages/e1/fe/9c4361e7ba2927074360856db6135ef4904d505e9b3afbbcb073c4008328/scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb", size = 36703062, upload-time = "2025-05-08T16:08:09.558Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8e/038ccfe29d272b30086b25a4960f757f97122cb2ec42e62b460d02fe98e9/scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4", size = 36393132, upload-time = "2025-05-08T16:08:15.34Z" }, - { url = "https://files.pythonhosted.org/packages/10/7e/5c12285452970be5bdbe8352c619250b97ebf7917d7a9a9e96b8a8140f17/scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5", size = 38979503, upload-time = "2025-05-08T16:08:21.513Z" }, - { url = "https://files.pythonhosted.org/packages/81/06/0a5e5349474e1cbc5757975b21bd4fad0e72ebf138c5592f191646154e06/scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca", size = 40308097, upload-time = "2025-05-08T16:08:27.627Z" }, -] - -[[package]] -name = "scipy" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'win32'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform == 'emscripten'", - "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/75/b4ce781849931fef6fd529afa6b63711d5a733065722d0c3e2724af9e40a/scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec", size = 31613675, upload-time = "2026-02-23T00:16:00.13Z" }, - { url = "https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696", size = 28162057, upload-time = "2026-02-23T00:16:09.456Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ee/18146b7757ed4976276b9c9819108adbc73c5aad636e5353e20746b73069/scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee", size = 20334032, upload-time = "2026-02-23T00:16:17.358Z" }, - { url = "https://files.pythonhosted.org/packages/ec/e6/cef1cf3557f0c54954198554a10016b6a03b2ec9e22a4e1df734936bd99c/scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd", size = 22709533, upload-time = "2026-02-23T00:16:25.791Z" }, - { url = "https://files.pythonhosted.org/packages/4d/60/8804678875fc59362b0fb759ab3ecce1f09c10a735680318ac30da8cd76b/scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c", size = 33062057, upload-time = "2026-02-23T00:16:36.931Z" }, - { url = "https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4", size = 35349300, upload-time = "2026-02-23T00:16:49.108Z" }, - { url = "https://files.pythonhosted.org/packages/b4/3d/7ccbbdcbb54c8fdc20d3b6930137c782a163fa626f0aef920349873421ba/scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444", size = 35127333, upload-time = "2026-02-23T00:17:01.293Z" }, - { url = "https://files.pythonhosted.org/packages/e8/19/f926cb11c42b15ba08e3a71e376d816ac08614f769b4f47e06c3580c836a/scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082", size = 37741314, upload-time = "2026-02-23T00:17:12.576Z" }, - { url = "https://files.pythonhosted.org/packages/95/da/0d1df507cf574b3f224ccc3d45244c9a1d732c81dcb26b1e8a766ae271a8/scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff", size = 36607512, upload-time = "2026-02-23T00:17:23.424Z" }, - { url = "https://files.pythonhosted.org/packages/68/7f/bdd79ceaad24b671543ffe0ef61ed8e659440eb683b66f033454dcee90eb/scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d", size = 24599248, upload-time = "2026-02-23T00:17:34.561Z" }, - { url = "https://files.pythonhosted.org/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8", size = 31610954, upload-time = "2026-02-23T00:17:49.855Z" }, - { url = "https://files.pythonhosted.org/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76", size = 28172662, upload-time = "2026-02-23T00:18:01.64Z" }, - { url = "https://files.pythonhosted.org/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086", size = 20344366, upload-time = "2026-02-23T00:18:12.015Z" }, - { url = "https://files.pythonhosted.org/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b", size = 22704017, upload-time = "2026-02-23T00:18:21.502Z" }, - { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, - { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, - { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, - { url = "https://files.pythonhosted.org/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87", size = 36549682, upload-time = "2026-02-23T00:19:07.67Z" }, - { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" }, - { url = "https://files.pythonhosted.org/packages/76/27/07ee1b57b65e92645f219b37148a7e7928b82e2b5dbeccecb4dff7c64f0b/scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c", size = 31590199, upload-time = "2026-02-23T00:19:17.192Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ae/db19f8ab842e9b724bf5dbb7db29302a91f1e55bc4d04b1025d6d605a2c5/scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f", size = 28154001, upload-time = "2026-02-23T00:19:22.241Z" }, - { url = "https://files.pythonhosted.org/packages/5b/58/3ce96251560107b381cbd6e8413c483bbb1228a6b919fa8652b0d4090e7f/scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d", size = 20325719, upload-time = "2026-02-23T00:19:26.329Z" }, - { url = "https://files.pythonhosted.org/packages/b2/83/15087d945e0e4d48ce2377498abf5ad171ae013232ae31d06f336e64c999/scipy-1.17.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b", size = 22683595, upload-time = "2026-02-23T00:19:30.304Z" }, - { url = "https://files.pythonhosted.org/packages/b4/e0/e58fbde4a1a594c8be8114eb4aac1a55bcd6587047efc18a61eb1f5c0d30/scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6", size = 32896429, upload-time = "2026-02-23T00:19:35.536Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5f/f17563f28ff03c7b6799c50d01d5d856a1d55f2676f537ca8d28c7f627cd/scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464", size = 35203952, upload-time = "2026-02-23T00:19:42.259Z" }, - { url = "https://files.pythonhosted.org/packages/8d/a5/9afd17de24f657fdfe4df9a3f1ea049b39aef7c06000c13db1530d81ccca/scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950", size = 34979063, upload-time = "2026-02-23T00:19:47.547Z" }, - { url = "https://files.pythonhosted.org/packages/8b/13/88b1d2384b424bf7c924f2038c1c409f8d88bb2a8d49d097861dd64a57b2/scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369", size = 37598449, upload-time = "2026-02-23T00:19:53.238Z" }, - { url = "https://files.pythonhosted.org/packages/35/e5/d6d0e51fc888f692a35134336866341c08655d92614f492c6860dc45bb2c/scipy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448", size = 36510943, upload-time = "2026-02-23T00:20:50.89Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fd/3be73c564e2a01e690e19cc618811540ba5354c67c8680dce3281123fb79/scipy-1.17.1-cp313-cp313-win_arm64.whl", hash = "sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87", size = 24545621, upload-time = "2026-02-23T00:20:55.871Z" }, - { url = "https://files.pythonhosted.org/packages/6f/6b/17787db8b8114933a66f9dcc479a8272e4b4da75fe03b0c282f7b0ade8cd/scipy-1.17.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a", size = 31936708, upload-time = "2026-02-23T00:19:58.694Z" }, - { url = "https://files.pythonhosted.org/packages/38/2e/524405c2b6392765ab1e2b722a41d5da33dc5c7b7278184a8ad29b6cb206/scipy-1.17.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0", size = 28570135, upload-time = "2026-02-23T00:20:03.934Z" }, - { url = "https://files.pythonhosted.org/packages/fd/c3/5bd7199f4ea8556c0c8e39f04ccb014ac37d1468e6cfa6a95c6b3562b76e/scipy-1.17.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce", size = 20741977, upload-time = "2026-02-23T00:20:07.935Z" }, - { url = "https://files.pythonhosted.org/packages/d9/b8/8ccd9b766ad14c78386599708eb745f6b44f08400a5fd0ade7cf89b6fc93/scipy-1.17.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6", size = 23029601, upload-time = "2026-02-23T00:20:12.161Z" }, - { url = "https://files.pythonhosted.org/packages/6d/a0/3cb6f4d2fb3e17428ad2880333cac878909ad1a89f678527b5328b93c1d4/scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e", size = 33019667, upload-time = "2026-02-23T00:20:17.208Z" }, - { url = "https://files.pythonhosted.org/packages/f3/c3/2d834a5ac7bf3a0c806ad1508efc02dda3c8c61472a56132d7894c312dea/scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475", size = 35264159, upload-time = "2026-02-23T00:20:23.087Z" }, - { url = "https://files.pythonhosted.org/packages/4d/77/d3ed4becfdbd217c52062fafe35a72388d1bd82c2d0ba5ca19d6fcc93e11/scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50", size = 35102771, upload-time = "2026-02-23T00:20:28.636Z" }, - { url = "https://files.pythonhosted.org/packages/bd/12/d19da97efde68ca1ee5538bb261d5d2c062f0c055575128f11a2730e3ac1/scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca", size = 37665910, upload-time = "2026-02-23T00:20:34.743Z" }, - { url = "https://files.pythonhosted.org/packages/06/1c/1172a88d507a4baaf72c5a09bb6c018fe2ae0ab622e5830b703a46cc9e44/scipy-1.17.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c", size = 36562980, upload-time = "2026-02-23T00:20:40.575Z" }, - { url = "https://files.pythonhosted.org/packages/70/b0/eb757336e5a76dfa7911f63252e3b7d1de00935d7705cf772db5b45ec238/scipy-1.17.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49", size = 24856543, upload-time = "2026-02-23T00:20:45.313Z" }, - { url = "https://files.pythonhosted.org/packages/cf/83/333afb452af6f0fd70414dc04f898647ee1423979ce02efa75c3b0f2c28e/scipy-1.17.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717", size = 31584510, upload-time = "2026-02-23T00:21:01.015Z" }, - { url = "https://files.pythonhosted.org/packages/ed/a6/d05a85fd51daeb2e4ea71d102f15b34fedca8e931af02594193ae4fd25f7/scipy-1.17.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9", size = 28170131, upload-time = "2026-02-23T00:21:05.888Z" }, - { url = "https://files.pythonhosted.org/packages/db/7b/8624a203326675d7746a254083a187398090a179335b2e4a20e2ddc46e83/scipy-1.17.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b", size = 20342032, upload-time = "2026-02-23T00:21:09.904Z" }, - { url = "https://files.pythonhosted.org/packages/c9/35/2c342897c00775d688d8ff3987aced3426858fd89d5a0e26e020b660b301/scipy-1.17.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866", size = 22678766, upload-time = "2026-02-23T00:21:14.313Z" }, - { url = "https://files.pythonhosted.org/packages/ef/f2/7cdb8eb308a1a6ae1e19f945913c82c23c0c442a462a46480ce487fdc0ac/scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350", size = 32957007, upload-time = "2026-02-23T00:21:19.663Z" }, - { url = "https://files.pythonhosted.org/packages/0b/2e/7eea398450457ecb54e18e9d10110993fa65561c4f3add5e8eccd2b9cd41/scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118", size = 35221333, upload-time = "2026-02-23T00:21:25.278Z" }, - { url = "https://files.pythonhosted.org/packages/d9/77/5b8509d03b77f093a0d52e606d3c4f79e8b06d1d38c441dacb1e26cacf46/scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068", size = 35042066, upload-time = "2026-02-23T00:21:31.358Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/18f80fb99df40b4070328d5ae5c596f2f00fffb50167e31439e932f29e7d/scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118", size = 37612763, upload-time = "2026-02-23T00:21:37.247Z" }, - { url = "https://files.pythonhosted.org/packages/4b/39/f0e8ea762a764a9dc52aa7dabcfad51a354819de1f0d4652b6a1122424d6/scipy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19", size = 37290984, upload-time = "2026-02-23T00:22:35.023Z" }, - { url = "https://files.pythonhosted.org/packages/7c/56/fe201e3b0f93d1a8bcf75d3379affd228a63d7e2d80ab45467a74b494947/scipy-1.17.1-cp314-cp314-win_arm64.whl", hash = "sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293", size = 25192877, upload-time = "2026-02-23T00:22:39.798Z" }, - { url = "https://files.pythonhosted.org/packages/96/ad/f8c414e121f82e02d76f310f16db9899c4fcde36710329502a6b2a3c0392/scipy-1.17.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6", size = 31949750, upload-time = "2026-02-23T00:21:42.289Z" }, - { url = "https://files.pythonhosted.org/packages/7c/b0/c741e8865d61b67c81e255f4f0a832846c064e426636cd7de84e74d209be/scipy-1.17.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1", size = 28585858, upload-time = "2026-02-23T00:21:47.706Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1b/3985219c6177866628fa7c2595bfd23f193ceebbe472c98a08824b9466ff/scipy-1.17.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39", size = 20757723, upload-time = "2026-02-23T00:21:52.039Z" }, - { url = "https://files.pythonhosted.org/packages/c0/19/2a04aa25050d656d6f7b9e7b685cc83d6957fb101665bfd9369ca6534563/scipy-1.17.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca", size = 23043098, upload-time = "2026-02-23T00:21:56.185Z" }, - { url = "https://files.pythonhosted.org/packages/86/f1/3383beb9b5d0dbddd030335bf8a8b32d4317185efe495374f134d8be6cce/scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad", size = 33030397, upload-time = "2026-02-23T00:22:01.404Z" }, - { url = "https://files.pythonhosted.org/packages/41/68/8f21e8a65a5a03f25a79165ec9d2b28c00e66dc80546cf5eb803aeeff35b/scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a", size = 35281163, upload-time = "2026-02-23T00:22:07.024Z" }, - { url = "https://files.pythonhosted.org/packages/84/8d/c8a5e19479554007a5632ed7529e665c315ae7492b4f946b0deb39870e39/scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4", size = 35116291, upload-time = "2026-02-23T00:22:12.585Z" }, - { url = "https://files.pythonhosted.org/packages/52/52/e57eceff0e342a1f50e274264ed47497b59e6a4e3118808ee58ddda7b74a/scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2", size = 37682317, upload-time = "2026-02-23T00:22:18.513Z" }, - { url = "https://files.pythonhosted.org/packages/11/2f/b29eafe4a3fbc3d6de9662b36e028d5f039e72d345e05c250e121a230dd4/scipy-1.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484", size = 37345327, upload-time = "2026-02-23T00:22:24.442Z" }, - { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, -] - -[[package]] -name = "seaborn" -version = "0.13.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "matplotlib", marker = "python_full_version < '3.13'" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696, upload-time = "2024-01-25T13:21:52.551Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914, upload-time = "2024-01-25T13:21:49.598Z" }, -] - [[package]] name = "setuptools" -version = "82.0.1" +version = "83.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] [[package]] @@ -4141,18 +3453,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] -[[package]] -name = "smart-open" -version = "7.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/65/3ada667d32675399001bf022ad3d9f3989b57101351ebc71d6fbe2384634/smart_open-7.6.1.tar.gz", hash = "sha256:4347996e7ba21db7cd1e059632e0b30395407e4f6c660d2ddffc8f2a9ae5f990", size = 54754, upload-time = "2026-05-09T06:23:37.06Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/78/0f68b93564b8c6b6987a0696c582ba2591a381ab2f733a501909e949f241/smart_open-7.6.1-py3-none-any.whl", hash = "sha256:b4de6aebef023aca91cc9fb372052e1343ba3f152de215bd22391a663e3ddd21", size = 64845, upload-time = "2026-05-09T06:23:35.386Z" }, -] - [[package]] name = "sniffio" version = "1.3.1" @@ -4173,11 +3473,11 @@ wheels = [ [[package]] name = "soupsieve" -version = "2.8.3" +version = "2.9" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/ae/2d9c981590ed9999a0d91755b47fc74f74de286b0f5cee14c9269041e6c4/soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349", size = 118627, upload-time = "2026-01-20T04:27:02.457Z" } +sdist = { url = "https://files.pythonhosted.org/packages/80/f1/93422647dd7e461f23d254e6b2bfa687a85b53aeb4903fcdbb74474d4584/soupsieve-2.9.tar.gz", hash = "sha256:acee8417325c5653e1377dc31eccad59eb82cbc65942afe6174c53b3aaad63fc", size = 122122, upload-time = "2026-07-19T01:35:18.425Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d6/3185ab5ad1280319b31986898f3206dd7227cd75e293d4dba2a5e6bf27a0/soupsieve-2.9-py3-none-any.whl", hash = "sha256:a2b2c76d67df2382d245409fd71e321a571717e58463efa32ace87dcadac2c12", size = 37387, upload-time = "2026-07-19T01:35:17.106Z" }, ] [[package]] @@ -4206,53 +3506,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] -[[package]] -name = "statsmodels" -version = "0.14.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "packaging", marker = "python_full_version < '3.13'" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "patsy", marker = "python_full_version < '3.13'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/81/e8d74b34f85285f7335d30c5e3c2d7c0346997af9f3debf9a0a9a63de184/statsmodels-0.14.6.tar.gz", hash = "sha256:4d17873d3e607d398b85126cd4ed7aad89e4e9d89fc744cdab1af3189a996c2a", size = 20689085, upload-time = "2025-12-05T23:08:39.522Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/6d/9ec309a175956f88eb8420ac564297f37cf9b1f73f89db74da861052dc29/statsmodels-0.14.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f4ff0649a2df674c7ffb6fa1a06bffdb82a6adf09a48e90e000a15a6aaa734b0", size = 10142419, upload-time = "2025-12-05T19:27:35.625Z" }, - { url = "https://files.pythonhosted.org/packages/86/8f/338c5568315ec5bf3ac7cd4b71e34b98cb3b0f834919c0c04a0762f878a1/statsmodels-0.14.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:109012088b3e370080846ab053c76d125268631410142daad2f8c10770e8e8d9", size = 10022819, upload-time = "2025-12-05T19:27:49.385Z" }, - { url = "https://files.pythonhosted.org/packages/b0/77/5fc4cbc2d608f9b483b0675f82704a8bcd672962c379fe4d82100d388dbf/statsmodels-0.14.6-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e93bd5d220f3cb6fc5fc1bffd5b094966cab8ee99f6c57c02e95710513d6ac3f", size = 10118927, upload-time = "2025-12-05T23:07:51.256Z" }, - { url = "https://files.pythonhosted.org/packages/94/55/b86c861c32186403fe121d9ab27bc16d05839b170d92a978beb33abb995e/statsmodels-0.14.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:06eec42d682fdb09fe5d70a05930857efb141754ec5a5056a03304c1b5e32fd9", size = 10413015, upload-time = "2025-12-05T23:08:53.95Z" }, - { url = "https://files.pythonhosted.org/packages/f9/be/daf0dba729ccdc4176605f4a0fd5cfe71cdda671749dca10e74a732b8b1c/statsmodels-0.14.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0444e88557df735eda7db330806fe09d51c9f888bb1f5906cb3a61fb1a3ed4a8", size = 10441248, upload-time = "2025-12-05T23:09:09.353Z" }, - { url = "https://files.pythonhosted.org/packages/9a/1c/2e10b7c7cc44fa418272996bf0427b8016718fd62f995d9c1f7ab37adf35/statsmodels-0.14.6-cp310-cp310-win_amd64.whl", hash = "sha256:e83a9abe653835da3b37fb6ae04b45480c1de11b3134bd40b09717192a1456ea", size = 9583410, upload-time = "2025-12-05T19:28:02.086Z" }, - { url = "https://files.pythonhosted.org/packages/a9/4d/df4dd089b406accfc3bb5ee53ba29bb3bdf5ae61643f86f8f604baa57656/statsmodels-0.14.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6ad5c2810fc6c684254a7792bf1cbaf1606cdee2a253f8bd259c43135d87cfb4", size = 10121514, upload-time = "2025-12-05T19:28:16.521Z" }, - { url = "https://files.pythonhosted.org/packages/82/af/ec48daa7f861f993b91a0dcc791d66e1cf56510a235c5cbd2ab991a31d5c/statsmodels-0.14.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:341fa68a7403e10a95c7b6e41134b0da3a7b835ecff1eb266294408535a06eb6", size = 10003346, upload-time = "2025-12-05T19:28:29.568Z" }, - { url = "https://files.pythonhosted.org/packages/a9/2c/c8f7aa24cd729970728f3f98822fb45149adc216f445a9301e441f7ac760/statsmodels-0.14.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdf1dfe2a3ca56f5529118baf33a13efed2783c528f4a36409b46bbd2d9d48eb", size = 10129872, upload-time = "2025-12-05T23:09:25.724Z" }, - { url = "https://files.pythonhosted.org/packages/40/c6/9ae8e9b0721e9b6eb5f340c3a0ce8cd7cce4f66e03dd81f80d60f111987f/statsmodels-0.14.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3764ba8195c9baf0925a96da0743ff218067a269f01d155ca3558deed2658ca", size = 10381964, upload-time = "2025-12-05T23:09:41.326Z" }, - { url = "https://files.pythonhosted.org/packages/28/8c/cf3d30c8c2da78e2ad1f50ade8b7fabec3ff4cdfc56fbc02e097c4577f90/statsmodels-0.14.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9e8d2e519852adb1b420e018f5ac6e6684b2b877478adf7fda2cfdb58f5acb5d", size = 10409611, upload-time = "2025-12-05T23:09:57.131Z" }, - { url = "https://files.pythonhosted.org/packages/bf/cc/018f14ecb58c6cb89de9d52695740b7d1f5a982aa9ea312483ea3c3d5f77/statsmodels-0.14.6-cp311-cp311-win_amd64.whl", hash = "sha256:2738a00fca51196f5a7d44b06970ace6b8b30289839e4808d656f8a98e35faa7", size = 9580385, upload-time = "2025-12-05T19:28:42.778Z" }, - { url = "https://files.pythonhosted.org/packages/25/ce/308e5e5da57515dd7cab3ec37ea2d5b8ff50bef1fcc8e6d31456f9fae08e/statsmodels-0.14.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe76140ae7adc5ff0e60a3f0d56f4fffef484efa803c3efebf2fcd734d72ecb5", size = 10091932, upload-time = "2025-12-05T19:28:55.446Z" }, - { url = "https://files.pythonhosted.org/packages/05/30/affbabf3c27fb501ec7b5808230c619d4d1a4525c07301074eb4bda92fa9/statsmodels-0.14.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26d4f0ed3b31f3c86f83a92f5c1f5cbe63fc992cd8915daf28ca49be14463a1c", size = 9997345, upload-time = "2025-12-05T19:29:10.278Z" }, - { url = "https://files.pythonhosted.org/packages/48/f5/3a73b51e6450c31652c53a8e12e24eac64e3824be816c0c2316e7dbdcb7d/statsmodels-0.14.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8c00a42863e4f4733ac9d078bbfad816249c01451740e6f5053ecc7db6d6368", size = 10058649, upload-time = "2025-12-05T23:10:12.775Z" }, - { url = "https://files.pythonhosted.org/packages/81/68/dddd76117df2ef14c943c6bbb6618be5c9401280046f4ddfc9fb4596a1b8/statsmodels-0.14.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b58cf7474aa9e7e3b0771a66537148b2df9b5884fbf156096c0e6c1ff0469d", size = 10339446, upload-time = "2025-12-05T23:10:28.503Z" }, - { url = "https://files.pythonhosted.org/packages/56/4a/dce451c74c4050535fac1ec0c14b80706d8fc134c9da22db3c8a0ec62c33/statsmodels-0.14.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:81e7dcc5e9587f2567e52deaff5220b175bf2f648951549eae5fc9383b62bc37", size = 10368705, upload-time = "2025-12-05T23:10:44.339Z" }, - { url = "https://files.pythonhosted.org/packages/60/15/3daba2df40be8b8a9a027d7f54c8dedf24f0d81b96e54b52293f5f7e3418/statsmodels-0.14.6-cp312-cp312-win_amd64.whl", hash = "sha256:b5eb07acd115aa6208b4058211138393a7e6c2cf12b6f213ede10f658f6a714f", size = 9543991, upload-time = "2025-12-05T23:10:58.536Z" }, - { url = "https://files.pythonhosted.org/packages/81/59/a5aad5b0cc266f5be013db8cde563ac5d2a025e7efc0c328d83b50c72992/statsmodels-0.14.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:47ee7af083623d2091954fa71c7549b8443168f41b7c5dce66510274c50fd73e", size = 10072009, upload-time = "2025-12-05T23:11:14.021Z" }, - { url = "https://files.pythonhosted.org/packages/53/dd/d8cfa7922fc6dc3c56fa6c59b348ea7de829a94cd73208c6f8202dd33f17/statsmodels-0.14.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa60d82e29fcd0a736e86feb63a11d2380322d77a9369a54be8b0965a3985f71", size = 9980018, upload-time = "2025-12-05T23:11:30.907Z" }, - { url = "https://files.pythonhosted.org/packages/ee/77/0ec96803eba444efd75dba32f2ef88765ae3e8f567d276805391ec2c98c6/statsmodels-0.14.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ee7d595f5939cc20bf946faedcb5137d975f03ae080f300ebb4398f16a5bd4", size = 10060269, upload-time = "2025-12-05T23:11:46.338Z" }, - { url = "https://files.pythonhosted.org/packages/10/b9/fd41f1f6af13a1a1212a06bb377b17762feaa6d656947bf666f76300fc05/statsmodels-0.14.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:730f3297b26749b216a06e4327fe0be59b8d05f7d594fb6caff4287b69654589", size = 10324155, upload-time = "2025-12-05T23:12:01.805Z" }, - { url = "https://files.pythonhosted.org/packages/ee/0f/a6900e220abd2c69cd0a07e3ad26c71984be6061415a60e0f17b152ecf08/statsmodels-0.14.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f1c08befa85e93acc992b72a390ddb7bd876190f1360e61d10cf43833463bc9c", size = 10349765, upload-time = "2025-12-05T23:12:18.018Z" }, - { url = "https://files.pythonhosted.org/packages/98/08/b79f0c614f38e566eebbdcff90c0bcacf3c6ba7a5bbb12183c09c29ca400/statsmodels-0.14.6-cp313-cp313-win_amd64.whl", hash = "sha256:8021271a79f35b842c02a1794465a651a9d06ec2080f76ebc3b7adce77d08233", size = 9540043, upload-time = "2025-12-05T23:12:33.887Z" }, - { url = "https://files.pythonhosted.org/packages/71/de/09540e870318e0c7b58316561d417be45eff731263b4234fdd2eee3511a8/statsmodels-0.14.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:00781869991f8f02ad3610da6627fd26ebe262210287beb59761982a8fa88cae", size = 10069403, upload-time = "2025-12-05T23:12:48.424Z" }, - { url = "https://files.pythonhosted.org/packages/ab/f0/63c1bfda75dc53cee858006e1f46bd6d6f883853bea1b97949d0087766ca/statsmodels-0.14.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:73f305fbf31607b35ce919fae636ab8b80d175328ed38fdc6f354e813b86ee37", size = 9989253, upload-time = "2025-12-05T23:13:05.274Z" }, - { url = "https://files.pythonhosted.org/packages/c1/98/b0dfb4f542b2033a3341aa5f1bdd97024230a4ad3670c5b0839d54e3dcab/statsmodels-0.14.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e443e7077a6e2d3faeea72f5a92c9f12c63722686eb80bb40a0f04e4a7e267ad", size = 10090802, upload-time = "2025-12-05T23:13:20.653Z" }, - { url = "https://files.pythonhosted.org/packages/34/0e/2408735aca9e764643196212f9069912100151414dd617d39ffc72d77eee/statsmodels-0.14.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3414e40c073d725007a6603a18247ab7af3467e1af4a5e5a24e4c27bc26673b4", size = 10337587, upload-time = "2025-12-05T23:13:37.597Z" }, - { url = "https://files.pythonhosted.org/packages/0f/36/4d44f7035ab3c0b2b6a4c4ebb98dedf36246ccbc1b3e2f51ebcd7ac83abb/statsmodels-0.14.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a518d3f9889ef920116f9fa56d0338069e110f823926356946dae83bc9e33e19", size = 10363350, upload-time = "2025-12-05T23:13:53.08Z" }, - { url = "https://files.pythonhosted.org/packages/26/33/f1652d0c59fa51de18492ee2345b65372550501ad061daa38f950be390b6/statsmodels-0.14.6-cp314-cp314-win_amd64.whl", hash = "sha256:151b73e29f01fe619dbce7f66d61a356e9d1fe5e906529b78807df9189c37721", size = 9588010, upload-time = "2025-12-05T23:14:07.28Z" }, -] - [[package]] name = "stevedore" version = "5.8.0" @@ -4262,15 +3515,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/ac/19f9941c74add59d17694930ec8105d5eddeee4ce56dd8632b765ca16d6c/stevedore-5.8.0-py3-none-any.whl", hash = "sha256:88eede9e66ca80e34085b9174e2327da2c61ac91f24f70e41c3ad76e4bb4872b", size = 54553, upload-time = "2026-05-18T09:15:25.82Z" }, ] -[[package]] -name = "threadpoolctl" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, -] - [[package]] name = "tiktoken" version = "0.13.0" @@ -4337,7 +3581,7 @@ name = "tokenizers" version = "0.23.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub", marker = "python_full_version >= '3.11'" }, + { name = "huggingface-hub" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" } wheels = [ @@ -4928,10 +4172,10 @@ name = "typer" version = "0.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-doc", marker = "python_full_version >= '3.11'" }, - { name = "click", marker = "python_full_version >= '3.11'" }, - { name = "rich", marker = "python_full_version >= '3.11'" }, - { name = "shellingham", marker = "python_full_version >= '3.11'" }, + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276, upload-time = "2026-04-30T19:32:16.964Z" } wheels = [ @@ -4968,25 +4212,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, ] -[[package]] -name = "umap-learn" -version = "0.5.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numba", marker = "python_full_version < '3.13'" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "pynndescent", marker = "python_full_version < '3.13'" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.13'" }, - { name = "tqdm", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/ee/af4171241117f85c74b5ca6448ea1033cc28d599c13651d67289bacd4083/umap_learn-0.5.12.tar.gz", hash = "sha256:6aff02ecac5f2aad9f3c65ee518d7ae93e1a985ae38721fdcffceee4232c33c7", size = 96672, upload-time = "2026-04-08T20:03:54.012Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/98/f63318ccbe75c810011fe9233884c5d348d94d90005de1b79e5f93bef9c0/umap_learn-0.5.12-py3-none-any.whl", hash = "sha256:f2a85d2a2adcb52b541bed9b27a23ca169b56bb1b23283abeebfb8dfb8a42fe5", size = 91849, upload-time = "2026-04-08T20:03:52.561Z" }, -] - [[package]] name = "urllib3" version = "2.7.0" @@ -5001,8 +4226,8 @@ name = "uvicorn" version = "0.47.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "python_full_version < '3.11' or sys_platform != 'emscripten'" }, - { name = "h11", marker = "python_full_version < '3.11' or sys_platform != 'emscripten'" }, + { name = "click" }, + { name = "h11" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f6/b1/8e7077a8641086aea449e1b5752a570f1b5906c64e0a33cd6d93b63a066b/uvicorn-0.47.0.tar.gz", hash = "sha256:7c9a0ea1a9414106bbab7324609c162d8fa0cdcdcb703060987269d77c7bb533", size = 90582, upload-time = "2026-05-14T18:16:54.455Z" } @@ -5070,92 +4295,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/1b/9e33c09813d65e248f7f773119148a612516a4bea93e9c6f545f78455b7c/wheel-0.47.0-py3-none-any.whl", hash = "sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced", size = 32218, upload-time = "2026-04-22T15:51:26.296Z" }, ] -[[package]] -name = "wrapt" -version = "2.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2e/64/925f213fdcbb9baeb1530449ac71a4d57fc361c053d06bf78d0c5c7cd80c/wrapt-2.1.2.tar.gz", hash = "sha256:3996a67eecc2c68fd47b4e3c564405a5777367adfd9b8abb58387b63ee83b21e", size = 81678, upload-time = "2026-03-06T02:53:25.134Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/d2/387594fb592d027366645f3d7cc9b4d7ca7be93845fbaba6d835a912ef3c/wrapt-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4b7a86d99a14f76facb269dc148590c01aaf47584071809a70da30555228158c", size = 60669, upload-time = "2026-03-06T02:52:40.671Z" }, - { url = "https://files.pythonhosted.org/packages/c9/18/3f373935bc5509e7ac444c8026a56762e50c1183e7061797437ca96c12ce/wrapt-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a819e39017f95bf7aede768f75915635aa8f671f2993c036991b8d3bfe8dbb6f", size = 61603, upload-time = "2026-03-06T02:54:21.032Z" }, - { url = "https://files.pythonhosted.org/packages/c2/7a/32758ca2853b07a887a4574b74e28843919103194bb47001a304e24af62f/wrapt-2.1.2-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5681123e60aed0e64c7d44f72bbf8b4ce45f79d81467e2c4c728629f5baf06eb", size = 113632, upload-time = "2026-03-06T02:53:54.121Z" }, - { url = "https://files.pythonhosted.org/packages/1d/d5/eeaa38f670d462e97d978b3b0d9ce06d5b91e54bebac6fbed867809216e7/wrapt-2.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b8b28e97a44d21836259739ae76284e180b18abbb4dcfdff07a415cf1016c3e", size = 115644, upload-time = "2026-03-06T02:54:53.33Z" }, - { url = "https://files.pythonhosted.org/packages/e3/09/2a41506cb17affb0bdf9d5e2129c8c19e192b388c4c01d05e1b14db23c00/wrapt-2.1.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cef91c95a50596fcdc31397eb6955476f82ae8a3f5a8eabdc13611b60ee380ba", size = 112016, upload-time = "2026-03-06T02:54:43.274Z" }, - { url = "https://files.pythonhosted.org/packages/64/15/0e6c3f5e87caadc43db279724ee36979246d5194fa32fed489c73643ba59/wrapt-2.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dad63212b168de8569b1c512f4eac4b57f2c6934b30df32d6ee9534a79f1493f", size = 114823, upload-time = "2026-03-06T02:54:29.392Z" }, - { url = "https://files.pythonhosted.org/packages/56/b2/0ad17c8248f4e57bedf44938c26ec3ee194715f812d2dbbd9d7ff4be6c06/wrapt-2.1.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d307aa6888d5efab2c1cde09843d48c843990be13069003184b67d426d145394", size = 111244, upload-time = "2026-03-06T02:54:02.149Z" }, - { url = "https://files.pythonhosted.org/packages/ff/04/bcdba98c26f2c6522c7c09a726d5d9229120163493620205b2f76bd13c01/wrapt-2.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c87cf3f0c85e27b3ac7d9ad95da166bf8739ca215a8b171e8404a2d739897a45", size = 113307, upload-time = "2026-03-06T02:54:12.428Z" }, - { url = "https://files.pythonhosted.org/packages/0e/1b/5e2883c6bc14143924e465a6fc5a92d09eeabe35310842a481fb0581f832/wrapt-2.1.2-cp310-cp310-win32.whl", hash = "sha256:d1c5fea4f9fe3762e2b905fdd67df51e4be7a73b7674957af2d2ade71a5c075d", size = 57986, upload-time = "2026-03-06T02:54:26.823Z" }, - { url = "https://files.pythonhosted.org/packages/42/5a/4efc997bccadd3af5749c250b49412793bc41e13a83a486b2b54a33e240c/wrapt-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:d8f7740e1af13dff2684e4d56fe604a7e04d6c94e737a60568d8d4238b9a0c71", size = 60336, upload-time = "2026-03-06T02:54:18Z" }, - { url = "https://files.pythonhosted.org/packages/c1/f5/a2bb833e20181b937e87c242645ed5d5aa9c373006b0467bfe1a35c727d0/wrapt-2.1.2-cp310-cp310-win_arm64.whl", hash = "sha256:1c6cc827c00dc839350155f316f1f8b4b0c370f52b6a19e782e2bda89600c7dc", size = 58757, upload-time = "2026-03-06T02:53:51.545Z" }, - { url = "https://files.pythonhosted.org/packages/c7/81/60c4471fce95afa5922ca09b88a25f03c93343f759aae0f31fb4412a85c7/wrapt-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:96159a0ee2b0277d44201c3b5be479a9979cf154e8c82fa5df49586a8e7679bb", size = 60666, upload-time = "2026-03-06T02:52:58.934Z" }, - { url = "https://files.pythonhosted.org/packages/6b/be/80e80e39e7cb90b006a0eaf11c73ac3a62bbfb3068469aec15cc0bc795de/wrapt-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:98ba61833a77b747901e9012072f038795de7fc77849f1faa965464f3f87ff2d", size = 61601, upload-time = "2026-03-06T02:53:00.487Z" }, - { url = "https://files.pythonhosted.org/packages/b0/be/d7c88cd9293c859fc74b232abdc65a229bb953997995d6912fc85af18323/wrapt-2.1.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:767c0dbbe76cae2a60dd2b235ac0c87c9cccf4898aef8062e57bead46b5f6894", size = 114057, upload-time = "2026-03-06T02:52:44.08Z" }, - { url = "https://files.pythonhosted.org/packages/ea/25/36c04602831a4d685d45a93b3abea61eca7fe35dab6c842d6f5d570ef94a/wrapt-2.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c691a6bc752c0cc4711cc0c00896fcd0f116abc253609ef64ef930032821842", size = 116099, upload-time = "2026-03-06T02:54:56.74Z" }, - { url = "https://files.pythonhosted.org/packages/5c/4e/98a6eb417ef551dc277bec1253d5246b25003cf36fdf3913b65cb7657a56/wrapt-2.1.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f3b7d73012ea75aee5844de58c88f44cf62d0d62711e39da5a82824a7c4626a8", size = 112457, upload-time = "2026-03-06T02:53:52.842Z" }, - { url = "https://files.pythonhosted.org/packages/cb/a6/a6f7186a5297cad8ec53fd7578533b28f795fdf5372368c74bd7e6e9841c/wrapt-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:577dff354e7acd9d411eaf4bfe76b724c89c89c8fc9b7e127ee28c5f7bcb25b6", size = 115351, upload-time = "2026-03-06T02:53:32.684Z" }, - { url = "https://files.pythonhosted.org/packages/97/6f/06e66189e721dbebd5cf20e138acc4d1150288ce118462f2fcbff92d38db/wrapt-2.1.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3d7b6fd105f8b24e5bd23ccf41cb1d1099796524bcc6f7fbb8fe576c44befbc9", size = 111748, upload-time = "2026-03-06T02:53:08.455Z" }, - { url = "https://files.pythonhosted.org/packages/ef/43/4808b86f499a51370fbdbdfa6cb91e9b9169e762716456471b619fca7a70/wrapt-2.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:866abdbf4612e0b34764922ef8b1c5668867610a718d3053d59e24a5e5fcfc15", size = 113783, upload-time = "2026-03-06T02:53:02.02Z" }, - { url = "https://files.pythonhosted.org/packages/91/2c/a3f28b8fa7ac2cefa01cfcaca3471f9b0460608d012b693998cd61ef43df/wrapt-2.1.2-cp311-cp311-win32.whl", hash = "sha256:5a0a0a3a882393095573344075189eb2d566e0fd205a2b6414e9997b1b800a8b", size = 57977, upload-time = "2026-03-06T02:53:27.844Z" }, - { url = "https://files.pythonhosted.org/packages/3f/c3/2b1c7bd07a27b1db885a2fab469b707bdd35bddf30a113b4917a7e2139d2/wrapt-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:64a07a71d2730ba56f11d1a4b91f7817dc79bc134c11516b75d1921a7c6fcda1", size = 60336, upload-time = "2026-03-06T02:54:28.104Z" }, - { url = "https://files.pythonhosted.org/packages/ec/5c/76ece7b401b088daa6503d6264dd80f9a727df3e6042802de9a223084ea2/wrapt-2.1.2-cp311-cp311-win_arm64.whl", hash = "sha256:b89f095fe98bc12107f82a9f7d570dc83a0870291aeb6b1d7a7d35575f55d98a", size = 58756, upload-time = "2026-03-06T02:53:16.319Z" }, - { url = "https://files.pythonhosted.org/packages/4c/b6/1db817582c49c7fcbb7df6809d0f515af29d7c2fbf57eb44c36e98fb1492/wrapt-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ff2aad9c4cda28a8f0653fc2d487596458c2a3f475e56ba02909e950a9efa6a9", size = 61255, upload-time = "2026-03-06T02:52:45.663Z" }, - { url = "https://files.pythonhosted.org/packages/a2/16/9b02a6b99c09227c93cd4b73acc3678114154ec38da53043c0ddc1fba0dc/wrapt-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6433ea84e1cfacf32021d2a4ee909554ade7fd392caa6f7c13f1f4bf7b8e8748", size = 61848, upload-time = "2026-03-06T02:53:48.728Z" }, - { url = "https://files.pythonhosted.org/packages/af/aa/ead46a88f9ec3a432a4832dfedb84092fc35af2d0ba40cd04aea3889f247/wrapt-2.1.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c20b757c268d30d6215916a5fa8461048d023865d888e437fab451139cad6c8e", size = 121433, upload-time = "2026-03-06T02:54:40.328Z" }, - { url = "https://files.pythonhosted.org/packages/3a/9f/742c7c7cdf58b59085a1ee4b6c37b013f66ac33673a7ef4aaed5e992bc33/wrapt-2.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79847b83eb38e70d93dc392c7c5b587efe65b3e7afcc167aa8abd5d60e8761c8", size = 123013, upload-time = "2026-03-06T02:53:26.58Z" }, - { url = "https://files.pythonhosted.org/packages/e8/44/2c3dd45d53236b7ed7c646fcf212251dc19e48e599debd3926b52310fafb/wrapt-2.1.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f8fba1bae256186a83d1875b2b1f4e2d1242e8fac0f58ec0d7e41b26967b965c", size = 117326, upload-time = "2026-03-06T02:53:11.547Z" }, - { url = "https://files.pythonhosted.org/packages/74/e2/b17d66abc26bd96f89dec0ecd0ef03da4a1286e6ff793839ec431b9fae57/wrapt-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e3d3b35eedcf5f7d022291ecd7533321c4775f7b9cd0050a31a68499ba45757c", size = 121444, upload-time = "2026-03-06T02:54:09.5Z" }, - { url = "https://files.pythonhosted.org/packages/3c/62/e2977843fdf9f03daf1586a0ff49060b1b2fc7ff85a7ea82b6217c1ae36e/wrapt-2.1.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6f2c5390460de57fa9582bc8a1b7a6c86e1a41dfad74c5225fc07044c15cc8d1", size = 116237, upload-time = "2026-03-06T02:54:03.884Z" }, - { url = "https://files.pythonhosted.org/packages/88/dd/27fc67914e68d740bce512f11734aec08696e6b17641fef8867c00c949fc/wrapt-2.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7dfa9f2cf65d027b951d05c662cc99ee3bd01f6e4691ed39848a7a5fffc902b2", size = 120563, upload-time = "2026-03-06T02:53:20.412Z" }, - { url = "https://files.pythonhosted.org/packages/ec/9f/b750b3692ed2ef4705cb305bd68858e73010492b80e43d2a4faa5573cbe7/wrapt-2.1.2-cp312-cp312-win32.whl", hash = "sha256:eba8155747eb2cae4a0b913d9ebd12a1db4d860fc4c829d7578c7b989bd3f2f0", size = 58198, upload-time = "2026-03-06T02:53:37.732Z" }, - { url = "https://files.pythonhosted.org/packages/8e/b2/feecfe29f28483d888d76a48f03c4c4d8afea944dbee2b0cd3380f9df032/wrapt-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1c51c738d7d9faa0b3601708e7e2eda9bf779e1b601dce6c77411f2a1b324a63", size = 60441, upload-time = "2026-03-06T02:52:47.138Z" }, - { url = "https://files.pythonhosted.org/packages/44/e1/e328f605d6e208547ea9fd120804fcdec68536ac748987a68c47c606eea8/wrapt-2.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:c8e46ae8e4032792eb2f677dbd0d557170a8e5524d22acc55199f43efedd39bf", size = 58836, upload-time = "2026-03-06T02:53:22.053Z" }, - { url = "https://files.pythonhosted.org/packages/4c/7a/d936840735c828b38d26a854e85d5338894cda544cb7a85a9d5b8b9c4df7/wrapt-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:787fd6f4d67befa6fe2abdffcbd3de2d82dfc6fb8a6d850407c53332709d030b", size = 61259, upload-time = "2026-03-06T02:53:41.922Z" }, - { url = "https://files.pythonhosted.org/packages/5e/88/9a9b9a90ac8ca11c2fdb6a286cb3a1fc7dd774c00ed70929a6434f6bc634/wrapt-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4bdf26e03e6d0da3f0e9422fd36bcebf7bc0eeb55fdf9c727a09abc6b9fe472e", size = 61851, upload-time = "2026-03-06T02:52:48.672Z" }, - { url = "https://files.pythonhosted.org/packages/03/a9/5b7d6a16fd6533fed2756900fc8fc923f678179aea62ada6d65c92718c00/wrapt-2.1.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bbac24d879aa22998e87f6b3f481a5216311e7d53c7db87f189a7a0266dafffb", size = 121446, upload-time = "2026-03-06T02:54:14.013Z" }, - { url = "https://files.pythonhosted.org/packages/45/bb/34c443690c847835cfe9f892be78c533d4f32366ad2888972c094a897e39/wrapt-2.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16997dfb9d67addc2e3f41b62a104341e80cac52f91110dece393923c0ebd5ca", size = 123056, upload-time = "2026-03-06T02:54:10.829Z" }, - { url = "https://files.pythonhosted.org/packages/93/b9/ff205f391cb708f67f41ea148545f2b53ff543a7ac293b30d178af4d2271/wrapt-2.1.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:162e4e2ba7542da9027821cb6e7c5e068d64f9a10b5f15512ea28e954893a267", size = 117359, upload-time = "2026-03-06T02:53:03.623Z" }, - { url = "https://files.pythonhosted.org/packages/1f/3d/1ea04d7747825119c3c9a5e0874a40b33594ada92e5649347c457d982805/wrapt-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f29c827a8d9936ac320746747a016c4bc66ef639f5cd0d32df24f5eacbf9c69f", size = 121479, upload-time = "2026-03-06T02:53:45.844Z" }, - { url = "https://files.pythonhosted.org/packages/78/cc/ee3a011920c7a023b25e8df26f306b2484a531ab84ca5c96260a73de76c0/wrapt-2.1.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:a9dd9813825f7ecb018c17fd147a01845eb330254dff86d3b5816f20f4d6aaf8", size = 116271, upload-time = "2026-03-06T02:54:46.356Z" }, - { url = "https://files.pythonhosted.org/packages/98/fd/e5ff7ded41b76d802cf1191288473e850d24ba2e39a6ec540f21ae3b57cb/wrapt-2.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6f8dbdd3719e534860d6a78526aafc220e0241f981367018c2875178cf83a413", size = 120573, upload-time = "2026-03-06T02:52:50.163Z" }, - { url = "https://files.pythonhosted.org/packages/47/c5/242cae3b5b080cd09bacef0591691ba1879739050cc7c801ff35c8886b66/wrapt-2.1.2-cp313-cp313-win32.whl", hash = "sha256:5c35b5d82b16a3bc6e0a04349b606a0582bc29f573786aebe98e0c159bc48db6", size = 58205, upload-time = "2026-03-06T02:53:47.494Z" }, - { url = "https://files.pythonhosted.org/packages/12/69/c358c61e7a50f290958809b3c61ebe8b3838ea3e070d7aac9814f95a0528/wrapt-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:f8bc1c264d8d1cf5b3560a87bbdd31131573eb25f9f9447bb6252b8d4c44a3a1", size = 60452, upload-time = "2026-03-06T02:53:30.038Z" }, - { url = "https://files.pythonhosted.org/packages/8e/66/c8a6fcfe321295fd8c0ab1bd685b5a01462a9b3aa2f597254462fc2bc975/wrapt-2.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:3beb22f674550d5634642c645aba4c72a2c66fb185ae1aebe1e955fae5a13baf", size = 58842, upload-time = "2026-03-06T02:52:52.114Z" }, - { url = "https://files.pythonhosted.org/packages/da/55/9c7052c349106e0b3f17ae8db4b23a691a963c334de7f9dbd60f8f74a831/wrapt-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0fc04bc8664a8bc4c8e00b37b5355cffca2535209fba1abb09ae2b7c76ddf82b", size = 63075, upload-time = "2026-03-06T02:53:19.108Z" }, - { url = "https://files.pythonhosted.org/packages/09/a8/ce7b4006f7218248dd71b7b2b732d0710845a0e49213b18faef64811ffef/wrapt-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a9b9d50c9af998875a1482a038eb05755dfd6fe303a313f6a940bb53a83c3f18", size = 63719, upload-time = "2026-03-06T02:54:33.452Z" }, - { url = "https://files.pythonhosted.org/packages/e4/e5/2ca472e80b9e2b7a17f106bb8f9df1db11e62101652ce210f66935c6af67/wrapt-2.1.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2d3ff4f0024dd224290c0eabf0240f1bfc1f26363431505fb1b0283d3b08f11d", size = 152643, upload-time = "2026-03-06T02:52:42.721Z" }, - { url = "https://files.pythonhosted.org/packages/36/42/30f0f2cefca9d9cbf6835f544d825064570203c3e70aa873d8ae12e23791/wrapt-2.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3278c471f4468ad544a691b31bb856374fbdefb7fee1a152153e64019379f015", size = 158805, upload-time = "2026-03-06T02:54:25.441Z" }, - { url = "https://files.pythonhosted.org/packages/bb/67/d08672f801f604889dcf58f1a0b424fe3808860ede9e03affc1876b295af/wrapt-2.1.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a8914c754d3134a3032601c6984db1c576e6abaf3fc68094bb8ab1379d75ff92", size = 145990, upload-time = "2026-03-06T02:53:57.456Z" }, - { url = "https://files.pythonhosted.org/packages/68/a7/fd371b02e73babec1de6ade596e8cd9691051058cfdadbfd62a5898f3295/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ff95d4264e55839be37bafe1536db2ab2de19da6b65f9244f01f332b5286cfbf", size = 155670, upload-time = "2026-03-06T02:54:55.309Z" }, - { url = "https://files.pythonhosted.org/packages/86/2d/9fe0095dfdb621009f40117dcebf41d7396c2c22dca6eac779f4c007b86c/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:76405518ca4e1b76fbb1b9f686cff93aebae03920cc55ceeec48ff9f719c5f67", size = 144357, upload-time = "2026-03-06T02:54:24.092Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b6/ec7b4a254abbe4cde9fa15c5d2cca4518f6b07d0f1b77d4ee9655e30280e/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c0be8b5a74c5824e9359b53e7e58bef71a729bacc82e16587db1c4ebc91f7c5a", size = 150269, upload-time = "2026-03-06T02:53:31.268Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6b/2fabe8ebf148f4ee3c782aae86a795cc68ffe7d432ef550f234025ce0cfa/wrapt-2.1.2-cp313-cp313t-win32.whl", hash = "sha256:f01277d9a5fc1862f26f7626da9cf443bebc0abd2f303f41c5e995b15887dabd", size = 59894, upload-time = "2026-03-06T02:54:15.391Z" }, - { url = "https://files.pythonhosted.org/packages/ca/fb/9ba66fc2dedc936de5f8073c0217b5d4484e966d87723415cc8262c5d9c2/wrapt-2.1.2-cp313-cp313t-win_amd64.whl", hash = "sha256:84ce8f1c2104d2f6daa912b1b5b039f331febfeee74f8042ad4e04992bd95c8f", size = 63197, upload-time = "2026-03-06T02:54:41.943Z" }, - { url = "https://files.pythonhosted.org/packages/c0/1c/012d7423c95d0e337117723eb8ecf73c622ce15a97847e84cf3f8f26cd7e/wrapt-2.1.2-cp313-cp313t-win_arm64.whl", hash = "sha256:a93cd767e37faeddbe07d8fc4212d5cba660af59bdb0f6372c93faaa13e6e679", size = 60363, upload-time = "2026-03-06T02:54:48.093Z" }, - { url = "https://files.pythonhosted.org/packages/39/25/e7ea0b417db02bb796182a5316398a75792cd9a22528783d868755e1f669/wrapt-2.1.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1370e516598854e5b4366e09ce81e08bfe94d42b0fd569b88ec46cc56d9164a9", size = 61418, upload-time = "2026-03-06T02:53:55.706Z" }, - { url = "https://files.pythonhosted.org/packages/ec/0f/fa539e2f6a770249907757eaeb9a5ff4deb41c026f8466c1c6d799088a9b/wrapt-2.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6de1a3851c27e0bd6a04ca993ea6f80fc53e6c742ee1601f486c08e9f9b900a9", size = 61914, upload-time = "2026-03-06T02:52:53.37Z" }, - { url = "https://files.pythonhosted.org/packages/53/37/02af1867f5b1441aaeda9c82deed061b7cd1372572ddcd717f6df90b5e93/wrapt-2.1.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:de9f1a2bbc5ac7f6012ec24525bdd444765a2ff64b5985ac6e0692144838542e", size = 120417, upload-time = "2026-03-06T02:54:30.74Z" }, - { url = "https://files.pythonhosted.org/packages/c3/b7/0138a6238c8ba7476c77cf786a807f871672b37f37a422970342308276e7/wrapt-2.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:970d57ed83fa040d8b20c52fe74a6ae7e3775ae8cff5efd6a81e06b19078484c", size = 122797, upload-time = "2026-03-06T02:54:51.539Z" }, - { url = "https://files.pythonhosted.org/packages/e1/ad/819ae558036d6a15b7ed290d5b14e209ca795dd4da9c58e50c067d5927b0/wrapt-2.1.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3969c56e4563c375861c8df14fa55146e81ac11c8db49ea6fb7f2ba58bc1ff9a", size = 117350, upload-time = "2026-03-06T02:54:37.651Z" }, - { url = "https://files.pythonhosted.org/packages/8b/2d/afc18dc57a4600a6e594f77a9ae09db54f55ba455440a54886694a84c71b/wrapt-2.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:57d7c0c980abdc5f1d98b11a2aa3bb159790add80258c717fa49a99921456d90", size = 121223, upload-time = "2026-03-06T02:54:35.221Z" }, - { url = "https://files.pythonhosted.org/packages/b9/5b/5ec189b22205697bc56eb3b62aed87a1e0423e9c8285d0781c7a83170d15/wrapt-2.1.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:776867878e83130c7a04237010463372e877c1c994d449ca6aaafeab6aab2586", size = 116287, upload-time = "2026-03-06T02:54:19.654Z" }, - { url = "https://files.pythonhosted.org/packages/f7/2d/f84939a7c9b5e6cdd8a8d0f6a26cabf36a0f7e468b967720e8b0cd2bdf69/wrapt-2.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fab036efe5464ec3291411fabb80a7a39e2dd80bae9bcbeeca5087fdfa891e19", size = 119593, upload-time = "2026-03-06T02:54:16.697Z" }, - { url = "https://files.pythonhosted.org/packages/0b/fe/ccd22a1263159c4ac811ab9374c061bcb4a702773f6e06e38de5f81a1bdc/wrapt-2.1.2-cp314-cp314-win32.whl", hash = "sha256:e6ed62c82ddf58d001096ae84ce7f833db97ae2263bff31c9b336ba8cfe3f508", size = 58631, upload-time = "2026-03-06T02:53:06.498Z" }, - { url = "https://files.pythonhosted.org/packages/65/0a/6bd83be7bff2e7efaac7b4ac9748da9d75a34634bbbbc8ad077d527146df/wrapt-2.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:467e7c76315390331c67073073d00662015bb730c566820c9ca9b54e4d67fd04", size = 60875, upload-time = "2026-03-06T02:53:50.252Z" }, - { url = "https://files.pythonhosted.org/packages/6c/c0/0b3056397fe02ff80e5a5d72d627c11eb885d1ca78e71b1a5c1e8c7d45de/wrapt-2.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:da1f00a557c66225d53b095a97eace0fc5349e3bfda28fa34ffae238978ee575", size = 59164, upload-time = "2026-03-06T02:53:59.128Z" }, - { url = "https://files.pythonhosted.org/packages/71/ed/5d89c798741993b2371396eb9d4634f009ff1ad8a6c78d366fe2883ea7a6/wrapt-2.1.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:62503ffbc2d3a69891cf29beeaccdb4d5e0a126e2b6a851688d4777e01428dbb", size = 63163, upload-time = "2026-03-06T02:52:54.873Z" }, - { url = "https://files.pythonhosted.org/packages/c6/8c/05d277d182bf36b0a13d6bd393ed1dec3468a25b59d01fba2dd70fe4d6ae/wrapt-2.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c7e6cd120ef837d5b6f860a6ea3745f8763805c418bb2f12eeb1fa6e25f22d22", size = 63723, upload-time = "2026-03-06T02:52:56.374Z" }, - { url = "https://files.pythonhosted.org/packages/f4/27/6c51ec1eff4413c57e72d6106bb8dec6f0c7cdba6503d78f0fa98767bcc9/wrapt-2.1.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3769a77df8e756d65fbc050333f423c01ae012b4f6731aaf70cf2bef61b34596", size = 152652, upload-time = "2026-03-06T02:53:23.79Z" }, - { url = "https://files.pythonhosted.org/packages/db/4c/d7dd662d6963fc7335bfe29d512b02b71cdfa23eeca7ab3ac74a67505deb/wrapt-2.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a76d61a2e851996150ba0f80582dd92a870643fa481f3b3846f229de88caf044", size = 158807, upload-time = "2026-03-06T02:53:35.742Z" }, - { url = "https://files.pythonhosted.org/packages/b4/4d/1e5eea1a78d539d346765727422976676615814029522c76b87a95f6bcdd/wrapt-2.1.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6f97edc9842cf215312b75fe737ee7c8adda75a89979f8e11558dfff6343cc4b", size = 146061, upload-time = "2026-03-06T02:52:57.574Z" }, - { url = "https://files.pythonhosted.org/packages/89/bc/62cabea7695cd12a288023251eeefdcb8465056ddaab6227cb78a2de005b/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4006c351de6d5007aa33a551f600404ba44228a89e833d2fadc5caa5de8edfbf", size = 155667, upload-time = "2026-03-06T02:53:39.422Z" }, - { url = "https://files.pythonhosted.org/packages/e9/99/6f2888cd68588f24df3a76572c69c2de28287acb9e1972bf0c83ce97dbc1/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a9372fc3639a878c8e7d87e1556fa209091b0a66e912c611e3f833e2c4202be2", size = 144392, upload-time = "2026-03-06T02:54:22.41Z" }, - { url = "https://files.pythonhosted.org/packages/40/51/1dfc783a6c57971614c48e361a82ca3b6da9055879952587bc99fe1a7171/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3144b027ff30cbd2fca07c0a87e67011adb717eb5f5bd8496325c17e454257a3", size = 150296, upload-time = "2026-03-06T02:54:07.848Z" }, - { url = "https://files.pythonhosted.org/packages/6c/38/cbb8b933a0201076c1f64fc42883b0023002bdc14a4964219154e6ff3350/wrapt-2.1.2-cp314-cp314t-win32.whl", hash = "sha256:3b8d15e52e195813efe5db8cec156eebe339aaf84222f4f4f051a6c01f237ed7", size = 60539, upload-time = "2026-03-06T02:54:00.594Z" }, - { url = "https://files.pythonhosted.org/packages/82/dd/e5176e4b241c9f528402cebb238a36785a628179d7d8b71091154b3e4c9e/wrapt-2.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:08ffa54146a7559f5b8df4b289b46d963a8e74ed16ba3687f99896101a3990c5", size = 63969, upload-time = "2026-03-06T02:54:39Z" }, - { url = "https://files.pythonhosted.org/packages/5c/99/79f17046cf67e4a95b9987ea129632ba8bcec0bc81f3fb3d19bdb0bd60cd/wrapt-2.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:72aaa9d0d8e4ed0e2e98019cea47a21f823c9dd4b43c7b77bba6679ffcca6a00", size = 60554, upload-time = "2026-03-06T02:53:14.132Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c7/8528ac2dfa2c1e6708f647df7ae144ead13f0a31146f43c7264b4942bf12/wrapt-2.1.2-py3-none-any.whl", hash = "sha256:b8fd6fa2b2c4e7621808f8c62e8317f4aae56e59721ad933bac5239d913cf0e8", size = 43993, upload-time = "2026-03-06T02:53:12.905Z" }, -] - [[package]] name = "yt-dlp" version = "2026.6.9"