feat: integrate public Helix embedded runtime

.github/workflows/benchmark.yml

@@ -0,0 +1,42 @@
+name: Helix acceptance benchmark
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Require the pinned public embedded wheel
+        run: python tools/check_helix_wheel.py
+      - name: Install isolated benchmark dependencies
+        run: |
+          python -m pip install -e . --no-deps
+          python -m pip install -r requirements-runtime.txt -r benchmarks/requirements-networkx.txt
+      - name: Run isolated NetworkX behavioral parity
+        env:
+          PYTHONPATH: .
+        run: python benchmarks/parity_networkx.py > parity-result.json
+      - name: Run 5k/15k and 20k/60k comparison
+        env:
+          PYTHONPATH: .
+        run: python benchmarks/helix_vs_networkx.py --out benchmark-result.json --check-gates
+      - uses: actions/upload-artifact@v4
+        with:
+          name: helix-vs-networkx-${{ matrix.os }}-${{ github.sha }}
+          path: |
+            benchmark-result.json
+            parity-result.json

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: CI
+name: Helix native CI
 
 on:
   push:
@@ -18,10 +18,8 @@ jobs:
     steps:
       - uses: actions/checkout@v6
         with:
-          # The audit-coverage, monolith-roundtrip, and always-on-roundtrip
-          # validators read blobs from origin/v8. A shallow checkout omits that
-          # ref, so fetch the full history here and the validators run for real
-          # (rather than skipping). The other jobs stay shallow.
+          # The heading audit reads a pinned pre-split v8 commit. A shallow
+          # checkout can omit it, so fetch full history for this job.
           fetch-depth: 0
 
       - name: Install uv
@@ -47,11 +45,23 @@ jobs:
         run: uv run --frozen python -m tools.skillgen --always-on-roundtrip
 
   test:
-    runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.10", "3.12"]
-
+        include:
+          - os: ubuntu-latest
+            python-version: "3.10"
+          - os: ubuntu-latest
+            python-version: "3.12"
+          - os: ubuntu-24.04-arm
+            python-version: "3.12"
+          - os: macos-latest
+            python-version: "3.12"
+          - os: windows-latest
+            python-version: "3.10"
+          - os: windows-latest
+            python-version: "3.12"
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v6
         with:
@@ -65,6 +75,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Require the pinned public embedded wheel
+        run: python tools/check_helix_wheel.py
+
       # --frozen installs straight from the committed uv.lock without re-resolving
       # or rewriting it, so CI never churns the lock.
       - name: Install dependencies
@@ -77,14 +90,38 @@ jobs:
         run: |
           uv run --frozen graphify --help
           uv run --frozen graphify install
+          uv build
+
+  quality:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: astral-sh/setup-uv@v8.1.0
+        with:
+          python-version: "3.12"
+
+      - run: uv sync --all-extras --frozen
+
+      - name: Ruff
+        run: uv run --frozen ruff check graphify tests tools benchmarks
+
+      - name: Pyright native integration surface
+        run: >-
+          uv run --frozen pyright graphify/helix graphify/operations.py graphify/cli.py
+          graphify/cluster.py graphify/analyze.py graphify/benchmark.py
+          graphify/global_graph.py graphify/impact.py graphify/affected.py
+          graphify/paths.py graphify/prs.py graphify/export.py graphify/watch.py
+
+      - name: Compile and whitespace checks
+        run: |
+          uv run --frozen python -m compileall -q graphify
+          git diff --check
 
   security-scan:
     # The dev deps include bandit and pip-audit. Run them in CI so a new
     # HIGH-severity finding or vulnerable dependency is caught on the PR that
     # introduces it, rather than at the next manual audit.
-    # Non-blocking for now (continue-on-error) to avoid breaking CI on
-    # pre-existing findings; remove continue-on-error after the initial
-    # cleanup pass.
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -98,9 +135,10 @@ jobs:
         run: uv sync --frozen
 
       - name: bandit (static security analysis)
-        continue-on-error: true
         run: uv run --frozen bandit -r graphify -ll
 
       - name: pip-audit (dependency vulnerabilities)
-        continue-on-error: true
-        run: uv run --frozen pip-audit --strict
+        run: |
+          uv export --quiet --frozen --all-extras --no-dev --no-emit-project \
+            --output-file audited-requirements.txt
+          uv run --frozen pip-audit --strict --requirement audited-requirements.txt

.github/workflows/release-graph.yml

@@ -37,12 +37,12 @@ jobs:
         run: uv run --frozen graphify cluster-only . --no-label
 
       - name: Generate HTML viewer
-        run: uv run --frozen graphify export html --graph graphify-out/graph.json
+        run: uv run --frozen graphify export html --graph graphify-out/graph.helix
 
       - name: Bundle release asset
         run: |
           mkdir -p dist-graph
-          cp graphify-out/graph.json dist-graph/
+          cp -R graphify-out/graph.helix dist-graph/
           cp graphify-out/graph.html dist-graph/
           [ -f graphify-out/GRAPH_REPORT.md ] && cp graphify-out/GRAPH_REPORT.md dist-graph/ || true
           tar -czf graphify-self-graph.tar.gz -C dist-graph .

.gitignore

@@ -12,6 +12,7 @@ build/
 .ruff_cache/
 *.so
 *.egg
+.DS_Store
 .graphify/
 graphify-out/
 .graphify_*.json

ARCHITECTURE.md

@@ -8,27 +8,53 @@ graphify is a Claude Code skill backed by a Python library. The skill orchestrat
 detect()  →  extract()  →  build_graph()  →  cluster()  →  analyze()  →  report()  →  export()
 ```
 
-Each stage is a single function in its own module. They communicate through plain Python dicts and NetworkX graphs - no shared state, no side effects outside `graphify-out/`.
+Extraction produces a transient, record-only `GraphBuildData`; every durable
+consumer receives a generation-safe `LoadedGraph` around Helix's native immutable
+graph. Embedded, on-disk Helix is the only durable graph/index store;
+NetworkX is neither imported nor installed in production.
+
+A build writes topology to an inactive generation, loads an immutable native
+snapshot for clustering and analysis, then stages topology and all durable state
+together. Counts and a SHA-256 checksum are verified before the active-generation
+pointer changes. A failed or interrupted build leaves the previous generation and
+its incremental hashes active.
 
 ## Module responsibilities
 
 | Module | Function | Input → Output |
 |--------|----------|----------------|
-| `detect.py` | `collect_files(root)` | directory → `[Path]` filtered list |
+| `detect.py` | `detect(root)` | directory → typed file lists and Helix-backed incremental state |
 | `extract.py` | `extract(path)` | file path → `{nodes, edges}` dict |
-| `build.py` | `build_graph(extractions)` | list of extraction dicts → `nx.Graph` |
-| `cluster.py` | `cluster(G)` | graph → graph with `community` attr on each node |
+| `build.py` | `build_from_json(extraction)` | extraction dict → transient record-only `GraphBuildData` |
+| `cluster.py` | `cluster(G)` | native snapshot → weighted Leiden membership |
 | `analyze.py` | `analyze(G)` | graph → analysis dict (god nodes, surprises, questions) |
 | `report.py` | `render_report(G, analysis)` | graph + analysis → GRAPH_REPORT.md string |
-| `export.py` | `export(G, out_dir, ...)` | graph → Obsidian vault, graph.json, graph.html, graph.svg |
+| `export.py` | `export(G, out_dir, ...)` | native snapshot → Obsidian vault, GraphML, Cypher, HTML, SVG |
 | `callflow_html.py` | `write_callflow_html(...)` | graphify-out files → Mermaid architecture/call-flow HTML |
 | `ingest.py` | `ingest(url, ...)` | URL → file saved to corpus dir |
 | `cache.py` | `check_semantic_cache / save_semantic_cache` | files → (cached, uncached) split |
 | `security.py` | validation helpers | URL / path / label → validated or raises |
 | `validate.py` | `validate_extraction(data)` | extraction dict → raises on schema errors |
-| `serve.py` | `start_server(graph_path)` | graph file path → MCP stdio server |
-| `watch.py` | `watch(root, flag_path)` | directory → writes flag file on change |
-| `benchmark.py` | `run_benchmark(graph_path)` | graph file → corpus vs subgraph token comparison |
+| `serve.py` | `serve(store_path)` | active Helix generation → reloadable MCP stdio server |
+| `watch.py` | `watch(root)` | changes → atomic embedded Helix rebuild |
+| `benchmark.py` | `run_benchmark(store_path)` | active Helix graph → corpus vs subgraph token comparison |
+
+## Durable generation schema
+
+Every active generation contains graph metadata, ordered typed nodes and edges,
+communities and names, analysis/report inputs, content and semantic hashes,
+extraction cache, extractor state, learning/provenance, and semantic-build
+metadata. There are no production topology, cache, label, analysis, or learning
+sidecars.
+
+Helix maintains unique equality indexes on `GraphifyNode(storage_key)` and
+`GraphifyControl(control_key)`. `storage_key` is the generation plus Helix's
+canonical typed external ID, so values such as `1`, `"1"`, and `true` cannot
+collide. Semantic relations are native edge labels; they are not duplicated in
+the edge attribute blob.
+
+Human-facing reports and explicit exports remain files. Existing Obsidian
+application configuration remains untouched.
 
 ## Extraction output schema
 
@@ -69,7 +95,7 @@ All external input passes through `graphify/security.py` before use:
 
 - URLs → `validate_url()` (http/https only) + `_NoFileRedirectHandler` (blocks file:// redirects)
 - Fetched content → `safe_fetch()` / `safe_fetch_text()` (size cap, timeout)
-- Graph file paths → `validate_graph_path()` (must resolve inside `graphify-out/`)
+- Helix store paths → `validate_store_path()` (must be an existing store directory)
 - Node labels → `sanitize_label()` (strips control chars, caps 256 chars, HTML-escapes)
 
 See `SECURITY.md` for the full threat model.
@@ -82,4 +108,8 @@ One test file per module under `tests/`. Run with:
 pytest tests/ -q
 ```
 
-All tests are pure unit tests - no network calls, no file system side effects outside `tmp_path`.
+The default suite includes unit, migration, corruption, atomicity, concurrency,
+native algorithm, and embedded close/reopen tests. The native tests require the
+pinned `helix-db-embedded` wheel. They are skipped in minimal source-only
+environments and mandatory in CI. NetworkX parity/performance is isolated under
+`benchmarks/` and is never a production dependency.

Dockerfile

@@ -2,10 +2,10 @@
 #
 # Build:  docker build -t graphify .
 # Run:    docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \
-#             /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET"
+#             /data/graph.helix --transport http --host 0.0.0.0 --api-key "$SECRET"
 #
 # Builds from source so the image includes the Streamable HTTP transport even
-# before it lands on PyPI. The graph.json is mounted at runtime (-v), never
+# before it lands on PyPI. The native Helix store is mounted at runtime (-v), never
 # baked into the image.
 FROM python:3.12-slim
 
@@ -22,4 +22,4 @@ USER graphify
 EXPOSE 8080
 
 ENTRYPOINT ["python", "-m", "graphify.serve"]
-CMD ["/data/graph.json", "--transport", "http", "--host", "0.0.0.0", "--port", "8080"]
+CMD ["/data/graph.helix", "--transport", "http", "--host", "0.0.0.0", "--port", "8080"]