diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..8de65289e --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,34 @@ +# Generated by simit. Manual edits will be reported as ci=drift. +name: CI + +on: + push: + branches: ["**"] + tags-ignore: ["**"] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + env: + NIX_CONFIG: "experimental-features = nix-command flakes" + XDG_CACHE_HOME: "/tmp/.cache" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v31 + + - name: Check generated flake wiring + run: nix run git+https://codeberg.org/caniko/simit.git -- init flake --check --diff + + - name: Check flake evaluation + run: nix flake check --no-build + + - name: Build check pytest + run: nix build .#checks.x86_64-linux.pytest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 548b40f22..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: CI - -on: - push: - branches: ["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "main"] - pull_request: - branches: ["v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "main"] - workflow_dispatch: - -jobs: - skillgen-check: - # Fast lint-style guard: the skill files under graphify/ are generated from - # the fragments in tools/skillgen/. This fails if someone hand-edited a - # generated file or forgot to re-run the generator and bless expected/, and it - # runs the build-time validators that guard per-host coverage, the file_type - # enum, the monolith round-trips, and the always-on round-trips. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - # The audit-coverage, monolith-roundtrip, and always-on-roundtrip - # validators read blobs from origin/v8. A shallow checkout omits that - # ref, so fetch the full history here and the validators run for real - # (rather than skipping). The other jobs stay shallow. - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - with: - python-version: "3.12" - - # --frozen keeps uv from re-resolving and rewriting uv.lock as a side - # effect of `uv run`; the lock is committed and must not churn in CI. - - name: Check generated skill artifacts are up to date - run: uv run --frozen python -m tools.skillgen --check - - - name: Audit per-host v8 coverage - run: uv run --frozen python -m tools.skillgen --audit-coverage - - - name: Check the file_type enum is a singleton - run: uv run --frozen python -m tools.skillgen --schema-singleton - - - name: Round-trip the monoliths against v8 - run: uv run --frozen python -m tools.skillgen --monolith-roundtrip - - - name: Round-trip the always-on blocks against v8 - run: uv run --frozen python -m tools.skillgen --always-on-roundtrip - - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.12"] - - steps: - - uses: actions/checkout@v6 - with: - # test_skillgen.py reads pre-split skill bodies from the immutable - # baseline commit via `git show`; a shallow checkout omits that history - # and the baseline tests fail. Full history mirrors the skillgen-check job. - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - with: - python-version: ${{ matrix.python-version }} - - # --frozen installs straight from the committed uv.lock without re-resolving - # or rewriting it, so CI never churns the lock. - - name: Install dependencies - run: uv sync --all-extras --frozen - - - name: Run tests - run: uv run --frozen pytest tests/ -q --tb=short - - - name: Verify install works end-to-end - run: | - uv run --frozen graphify --help - uv run --frozen graphify install - - security-scan: - # The dev deps include bandit and pip-audit. Run them in CI so a new - # HIGH-severity finding or vulnerable dependency is caught on the PR that - # introduces it, rather than at the next manual audit. - # Non-blocking for now (continue-on-error) to avoid breaking CI on - # pre-existing findings; remove continue-on-error after the initial - # cleanup pass. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - with: - python-version: "3.12" - - - name: Install dependencies - run: uv sync --frozen - - - name: bandit (static security analysis) - continue-on-error: true - run: uv run --frozen bandit -r graphify -ll - - - name: pip-audit (dependency vulnerabilities) - continue-on-error: true - run: uv run --frozen pip-audit --strict diff --git a/.gitignore b/.gitignore index 0a6775b2a..640aef9b3 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ docs/superpowers/ .vscode/ .kilo openspec/ +result # Local benchmark scripts — never commit scripts/run_k2_*.py scripts/llm.py diff --git a/README.md b/README.md index e73db3aa0..8129ab168 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ Every system ran on the same harness with the same model and budgets, scored by | Python | 3.10+ | `python --version` | [python.org](https://www.python.org/downloads/) | | uv *(recommended)* | any | `uv --version` | `curl -LsSf https://astral.sh/uv/install.sh \| sh` | | pipx *(alternative)* | any | `pipx --version` | `pip install pipx` | +| Nix *(alternative)* | 2.4+ (flakes enabled) | `nix --version` | [nixos.org](https://nixos.org/download/) | **macOS quick install (Homebrew):** ```bash @@ -163,6 +164,87 @@ pipx install graphifyy pip install graphifyy # may need PATH setup — see note below ``` +**Nix (flake):** + +Run directly without installing: +```bash +nix run github:caniko/graphify +``` + +To expose `graphify` as a package inside another flake, add it as an input and reference its default package: +```nix +{ + inputs = { + graphify.url = "github:caniko/graphify"; + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + }; + + outputs = { nixpkgs, graphify, ... }: { + let + system = "x86_64-linux"; + pkgs = nixpkgs.legacyPackages.${system}; + in { + devShells.${system}.default = pkgs.mkShell { + buildInputs = [ graphify.packages.${system}.default ]; + }; + }; + }; +} +``` + +The flake and GitHub Actions workflow are generated and checked by +[simit](https://codeberg.org/caniko/simit). After changing the flake outputs, +refresh the generated wiring and verify the same checks CI runs: + +```bash +simit init flake --check --diff +simit init ci --platform github --runtime nix --check --diff +nix flake check --no-build +nix build .#checks.x86_64-linux.pytest +``` + +Keep `simit.toml`, `nix/pre-commit.nix`, and `.github/workflows/ci.yaml` in +sync; the workflow intentionally builds the named `pytest` check instead of +duplicating a second Python dependency installation path. + +For a long-running NixOS deployment, import `graphify.nixosModules.default`. +The module uses the full runtime package (including MCP, PostgreSQL, graph +database exports, and every built-in LLM provider) while `packages.default` +stays lean for interactive AST-only use. Each named instance owns independent +state, source selection, extraction schedule, optional file watcher, HTTP MCP +listener, and Neo4j/FalkorDB sinks: + +```nix +{ + imports = [inputs.graphify.nixosModules.default]; + + services.graphify = { + enable = true; + instances.database = { + source.postgresql = { + enable = true; + host = "/run/postgresql"; + database = "app"; + user = "graphify"; + systemdService = "postgresql.service"; + }; + extraction.onCalendar = "daily"; + server.enable = true; # loopback HTTP MCP on port 8080 + }; + }; +} +``` + +PostgreSQL is a read-only schema input, not Graphify's persistence backend; +the resulting graph remains +`/var/lib/graphify//graphify-out/graph.json`. The reusable module +does not create database roles or grants. Host policy must provision a role +that can see the selected schema. Passwords, pgpass files, provider keys, MCP +keys, and graph-database passwords have file-backed options and are loaded as +systemd credentials instead of appearing on process command lines. + +--- + **Step 2 — register the skill with your AI assistant:** ```bash diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..b5a47d707 --- /dev/null +++ b/flake.lock @@ -0,0 +1,120 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1778716662, + "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1779560665, + "narHash": "sha256-tpyBcxPpcQb8ukyNF7DoCwfSY3VPsxHoYwj00Cayv5o=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "pyproject-build-systems": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-nix": [ + "pyproject-nix" + ], + "uv2nix": [ + "uv2nix" + ] + }, + "locked": { + "lastModified": 1779676664, + "narHash": "sha256-MbXylBTkWqVm8/VYjoULtMoVRgWBN1gSHbeRKsOsPlU=", + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "rev": "7bff980f37fc24e09dbc986643719900c139bf12", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "type": "github" + } + }, + "pyproject-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1778901413, + "narHash": "sha256-GSKXTAnFqRAMlZkJrIPcQMYf+lpMr66K3i60mB9STvc=", + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "rev": "a228447c3e179d477c1b6246ef3efa8cfe3c469a", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "pyproject-build-systems": "pyproject-build-systems", + "pyproject-nix": "pyproject-nix", + "uv2nix": "uv2nix" + } + }, + "uv2nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-nix": [ + "pyproject-nix" + ] + }, + "locked": { + "lastModified": 1779411315, + "narHash": "sha256-IMFlxeyClau51KplhhSRGhdGTvD/knShHdybP1UOTuk=", + "owner": "pyproject-nix", + "repo": "uv2nix", + "rev": "fdf2a76275d7a9c27deb5d2f2ab33526ac9052ff", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "uv2nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..8ac4b675e --- /dev/null +++ b/flake.nix @@ -0,0 +1,377 @@ +{ + description = "flake for graphify using uv2nix"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + + flake-parts = { + url = "github:hercules-ci/flake-parts"; + inputs.nixpkgs-lib.follows = "nixpkgs"; + }; + + pyproject-nix = { + url = "github:pyproject-nix/pyproject.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + uv2nix = { + url = "github:pyproject-nix/uv2nix"; + inputs.pyproject-nix.follows = "pyproject-nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + pyproject-build-systems = { + url = "github:pyproject-nix/build-system-pkgs"; + inputs.pyproject-nix.follows = "pyproject-nix"; + inputs.uv2nix.follows = "uv2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = inputs @ { + flake-parts, + pyproject-nix, + uv2nix, + pyproject-build-systems, + ... + }: + flake-parts.lib.mkFlake {inherit inputs;} { + systems = ["x86_64-linux" "aarch64-linux" "aarch64-darwin"]; + + flake.nixosModules.default = { + lib, + pkgs, + ... + }: { + imports = [./nix/nixos-module.nix]; + services.graphify.package = lib.mkDefault inputs.self.packages.${pkgs.stdenv.hostPlatform.system}.full; + }; + + perSystem = { + pkgs, + lib, + ... + }: let + pyproject = lib.importTOML ./pyproject.toml; + projectMeta = pyproject.project; + + workspace = uv2nix.lib.workspace.loadWorkspace {workspaceRoot = ./.;}; + + overlay = workspace.mkPyprojectOverlay { + sourcePreference = "wheel"; + }; + + editableOverlay = workspace.mkEditablePyprojectOverlay { + root = "$REPO_ROOT"; + }; + + python = pkgs.python312; + + baseSet = + (pkgs.callPackage pyproject-nix.build.packages { + inherit python; + }) + .overrideScope + ( + lib.composeManyExtensions [ + pyproject-build-systems.overlays.wheel + overlay + ] + ); + + pythonSet = baseSet.overrideScope (final: prev: { + # numba manylinux wheel dlopens libtbb.so at runtime; expose it so + # autoPatchelfHook (from pyproject-build-systems' wheel overlay) can + # resolve it on the rpath. + numba = prev.numba.overrideAttrs (old: { + buildInputs = (old.buildInputs or []) ++ [pkgs.tbb]; + }); + + # nuitka's sdist doesn't declare setuptools as a build dep. + nuitka = prev.nuitka.overrideAttrs (old: { + nativeBuildInputs = + (old.nativeBuildInputs or []) + ++ final.resolveBuildSystem {setuptools = [];}; + }); + + # jieba's sdist doesn't declare setuptools as a build dep. + jieba = prev.jieba.overrideAttrs (old: { + nativeBuildInputs = + (old.nativeBuildInputs or []) + ++ final.resolveBuildSystem {setuptools = [];}; + }); + + # tree-sitter-dm's sdist doesn't declare setuptools as a build dep. + tree-sitter-dm = prev.tree-sitter-dm.overrideAttrs (old: { + nativeBuildInputs = + (old.nativeBuildInputs or []) + ++ final.resolveBuildSystem {setuptools = [];}; + }); + + # Expose tests via passthru.tests so they can be wired into flake + # checks (mirrors the uv2nix testing pattern). + graphifyy = prev.graphifyy.overrideAttrs (old: { + passthru = + (old.passthru or {}) + // { + tests = let + # Virtualenv containing graphify plus the dev dependency + # group (which carries pytest and friends). + testVenv = final.mkVirtualEnv "graphify-test-env" (workspace.deps.default + // { + # The retry-cap tests exercise the OpenAI-compatible Ollama + # path, so include that optional extra in the test-only + # environment without pulling every runtime extra. + graphifyy = ["dev" "ollama"]; + }); + in + (old.passthru.tests or {}) + // { + pytest = pkgs.stdenv.mkDerivation { + name = "${final.graphifyy.name}-pytest"; + # Test the repository tree rather than the wheel source: + # skillgen's fixtures and extraction-spec fragments are + # intentionally repository assets, not package payload. + src = ./.; + nativeBuildInputs = [testVenv pkgs.git]; + dontConfigure = true; + + buildPhase = '' + runHook preBuild + # The Nix build sandbox sets HOME=/homeless-shelter + # which is unwritable; several tests (e.g. the Gemini + # install ones) call helpers that resolve paths via + # Path.home() when not project-scoped. Point HOME at a + # writable temp dir so those tests pass under + # `nix flake check`. + export HOME=''${PWD}/home + pytest + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + touch $out + runHook postInstall + ''; + }; + }; + }; + }); + }); + + editablePythonSet = pythonSet.overrideScope editableOverlay; + virtualenv = editablePythonSet.mkVirtualEnv "graphify-dev-env" workspace.deps.all; + + graphifyEnv = pythonSet.mkVirtualEnv "graphify-env" workspace.deps.default; + graphifyFullEnv = pythonSet.mkVirtualEnv "graphify-full-env" (workspace.deps.default + // { + graphifyy = ["all"]; + }); + + # Wrap a virtualenv so consumers receive stable public entry points while + # the environment remains available for smoke checks and composition. + mkGraphifyPackage = { + environment, + suffix ? "", + }: + pkgs.stdenv.mkDerivation { + pname = projectMeta.name + suffix; + version = projectMeta.version; + + dontUnpack = true; + dontBuild = true; + dontConfigure = true; + + nativeBuildInputs = [pkgs.makeWrapper]; + + installPhase = '' + mkdir -p $out/bin + makeWrapper ${environment}/bin/graphify $out/bin/graphify + if [ -x ${environment}/bin/graphify-mcp ]; then + makeWrapper ${environment}/bin/graphify-mcp $out/bin/graphify-mcp + fi + ''; + + passthru = { + graphifyEnv = environment; + }; + + meta = { + description = projectMeta.description; + homepage = projectMeta.urls.Homepage; + license = lib.licenses.mit; + mainProgram = "graphify"; + platforms = lib.platforms.unix; + }; + }; + + graphifyPackage = mkGraphifyPackage {environment = graphifyEnv;}; + graphifyFullPackage = mkGraphifyPackage { + environment = graphifyFullEnv; + suffix = "-full"; + }; + + moduleSample = inputs.nixpkgs.lib.nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + specialArgs.graphifyPackage = graphifyFullPackage; + modules = [ + ./nix/nixos-module.nix + { + system.stateVersion = "24.11"; + services.graphify = { + enable = true; + instances.postgres-only = { + source.postgresql = { + enable = true; + database = "catalog"; + }; + extraction.noCluster = true; + }; + instances.matrix = { + source = { + path = "/srv/source"; + cargo = true; + postgresql = { + enable = true; + host = "/run/postgresql"; + database = "app"; + user = "graphify"; + sslMode = "disable"; + pgpassFile = "/run/keys/pgpass"; + systemdService = "postgresql.service"; + }; + }; + extraction = { + mode = "deep"; + codeOnly = true; + noCluster = true; + dedup = true; + googleWorkspace = true; + global = true; + tag = "matrix"; + maxWorkers = 2; + tokenBudget = 4096; + maxConcurrency = 2; + apiTimeout = 30; + resolution = 1.25; + excludeHubs = 0.95; + excludes = ["vendor" "target"]; + timing = true; + onCalendar = "hourly"; + }; + llm = { + backend = "openai"; + model = "test-model"; + baseUrl = "http://127.0.0.1:8081/v1"; + apiKeyFile = "/run/keys/openai"; + }; + watch = { + enable = true; + debounce = 1.5; + }; + server = { + enable = true; + host = "0.0.0.0"; + port = 8080; + path = "/mcp"; + jsonResponse = true; + stateless = true; + sessionTimeout = 0; + apiKeyFile = "/run/keys/mcp"; + openFirewall = true; + }; + exports = { + neo4j = { + enable = true; + uri = "bolt://127.0.0.1:7687"; + passwordFile = "/run/keys/neo4j"; + }; + falkordb = { + enable = true; + uri = "falkordb://127.0.0.1:6379"; + onCalendar = "daily"; + }; + }; + environment.GRAPHIFY_MAX_RETRIES = "3"; + environmentFiles = ["/run/keys/graphify.env"]; + }; + }; + } + ]; + }; + in { + formatter = pkgs.writeShellApplication { + name = "graphify-nix-format"; + runtimeInputs = [pkgs.alejandra]; + text = '' + has_path=0 + for argument in "$@"; do + case "$argument" in + -*) ;; + *) has_path=1 ;; + esac + done + if [ "$has_path" -eq 0 ]; then + set -- "$@" . + fi + exec alejandra "$@" + ''; + }; + + devShells.default = pkgs.mkShell { + packages = [ + virtualenv + pkgs.uv + pkgs.python3Packages.pytest + ]; + env = { + UV_NO_SYNC = "1"; + UV_PYTHON = editablePythonSet.python.interpreter; + UV_PYTHON_DOWNLOADS = "never"; + UV_PROJECT_ENVIRONMENT = virtualenv.outPath; + VIRTUAL_ENV = virtualenv.outPath; + }; + + shellHook = '' + unset PYTHONPATH + export REPO_ROOT=$(git rev-parse --show-toplevel) + ''; + }; + + packages = { + default = graphifyPackage; + full = graphifyFullPackage; + }; + + checks = { + inherit (pythonSet.graphifyy.passthru.tests) pytest; + full-package = pkgs.runCommand "graphify-full-package-check" {} '' + test -x ${graphifyFullPackage}/bin/graphify + test -x ${graphifyFullPackage}/bin/graphify-mcp + ${graphifyFullEnv}/bin/python -c 'import anthropic, boto3, falkordb, mcp, neo4j, openai, psycopg' + touch $out + ''; + nixos-module = pkgs.runCommand "graphify-nixos-module-check" {} '' + test '${moduleSample.config.services.graphify.instances.matrix.source.postgresql.database}' = app + test '${toString moduleSample.config.services.graphify.instances.matrix.server.port}' = 8080 + test '${toString moduleSample.config.networking.firewall.allowedTCPPorts}' = 8080 + case ${lib.escapeShellArg (toString moduleSample.config.systemd.services.graphify-matrix-extract.serviceConfig.ExecStart)} in + *graphify-matrix-extract*) ;; + *) echo 'missing Graphify extract service' >&2; exit 1 ;; + esac + test '${moduleSample.config.services.graphify.instances.matrix.exports.neo4j.uri}' = 'bolt://127.0.0.1:7687' + test -n '${toString moduleSample.config.systemd.services.graphify-matrix-neo4j.serviceConfig.ExecStart}' + test '${toString moduleSample.config.systemd.timers.graphify-matrix-falkordb.timerConfig.OnCalendar}' = daily + touch $out + ''; + }; + + apps.default = { + type = "app"; + program = "${graphifyPackage}/bin/graphify"; + meta = graphifyPackage.meta; + }; + }; + }; +} diff --git a/graphify/cli.py b/graphify/cli.py index 774034220..7ca95199f 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -2550,6 +2550,12 @@ def _parse_float(name: str, raw: str) -> float: ) if not has_path: + # PostgreSQL-only extraction has no filesystem corpus to detect, but + # the common reporting/manifest path below still consumes the + # detection result. Keep the same shape as detect() returns so + # ``graphify extract --postgres DSN`` can continue through database + # introspection without reading an unbound local. + detection = {"files": {}, "unclassified": []} code_files = [] doc_files = [] paper_files = [] diff --git a/graphify/extract.py b/graphify/extract.py index bc10f7d7d..a030c6ee0 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -3486,7 +3486,14 @@ def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]: except OSError: return classes for cs_path in cs_files: - if any(_is_noise_dir(part) for part in cs_path.parts): + # Check only project-relative components. Absolute build sandboxes + # commonly mount sources below ``/build``; treating that ancestor as a + # project noise directory would hide every ViewModel from XAML linking. + try: + relative_parts = cs_path.relative_to(root).parts + except ValueError: + relative_parts = cs_path.parts + if any(_is_noise_dir(part) for part in relative_parts[:-1]): continue if patterns and _is_ignored(cs_path, root, patterns, _cache=ignore_cache): continue @@ -5077,9 +5084,10 @@ def _ignored(p: Path) -> bool: return bool(patterns and _is_ignored(p, ignore_root, patterns, _cache=ignore_cache)) if not follow_symlinks: - # The old rglob filter rejected paths with a noise component anywhere, - # including components of target itself — preserve that. - if any(_is_noise_dir(part) for part in target.parts): + # Only reject if the target directory *itself* is a noise dir (e.g. + # node_modules passed directly). Do NOT check ancestor path components + # — that would incorrectly exclude projects living inside .worktrees/. + if _is_noise_dir(target.name): return [] # When negation (!) patterns exist, skip directory-level ignore pruning # so negated files inside ignored dirs can still be reached (same diff --git a/nix/nixos-module.nix b/nix/nixos-module.nix new file mode 100644 index 000000000..b15a82b45 --- /dev/null +++ b/nix/nixos-module.nix @@ -0,0 +1,759 @@ +{ + config, + graphifyPackage ? null, + lib, + pkgs, + ... +}: let + inherit (lib) concatLists escapeShellArgs filterAttrs flatten mapAttrs' mapAttrsToList mkEnableOption mkIf mkMerge mkOption nameValuePair optional optionalAttrs optionalString types unique; + cfg = config.services.graphify; + + builtInBackends = [ + "azure" + "bedrock" + "claude" + "claude-cli" + "deepseek" + "gemini" + "kimi" + "ollama" + "openai" + ]; + + backendApiKeyVariables = { + azure = "AZURE_OPENAI_API_KEY"; + claude = "ANTHROPIC_API_KEY"; + deepseek = "DEEPSEEK_API_KEY"; + gemini = "GEMINI_API_KEY"; + kimi = "MOONSHOT_API_KEY"; + ollama = "OLLAMA_API_KEY"; + openai = "OPENAI_API_KEY"; + }; + + backendBaseUrlVariables = { + azure = "AZURE_OPENAI_ENDPOINT"; + claude = "ANTHROPIC_BASE_URL"; + deepseek = "DEEPSEEK_BASE_URL"; + gemini = "GEMINI_BASE_URL"; + kimi = "KIMI_BASE_URL"; + ollama = "OLLAMA_BASE_URL"; + openai = "OPENAI_BASE_URL"; + }; + + positiveInt = types.ints.positive; + nullablePositiveInt = types.nullOr positiveInt; + nullablePositiveNumber = types.nullOr (types.addCheck types.number (value: value > 0)); + + postgresOptions = {name, ...}: { + options = { + enable = mkEnableOption "PostgreSQL schema introspection for ${name}"; + + host = mkOption { + type = types.str; + default = "/run/postgresql"; + description = "libpq host name, address, or Unix socket directory."; + }; + + port = mkOption { + type = types.port; + default = 5432; + description = "PostgreSQL port."; + }; + + database = mkOption { + type = types.nullOr types.str; + default = null; + description = "Database whose schema Graphify introspects."; + }; + + user = mkOption { + type = types.str; + default = cfg.user; + defaultText = lib.literalExpression "config.services.graphify.user"; + description = "PostgreSQL role. Local peer authentication can use the Graphify system user."; + }; + + sslMode = mkOption { + type = types.enum ["disable" "allow" "prefer" "require" "verify-ca" "verify-full"]; + default = "prefer"; + description = "libpq SSL mode."; + }; + + pgpassFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Optional pgpass file loaded as a systemd credential; it never appears on argv."; + }; + + systemdService = mkOption { + type = types.nullOr types.str; + default = null; + example = "postgresql.service"; + description = "Optional local PostgreSQL unit required before extraction."; + }; + }; + }; + + extractionOptions = {name, ...}: { + options = { + enable = mkEnableOption "headless extraction for ${name}" // {default = true;}; + startAtBoot = mkOption { + type = types.bool; + default = true; + description = "Run extraction during multi-user startup."; + }; + onCalendar = mkOption { + type = types.nullOr types.str; + default = null; + example = "daily"; + description = "Optional systemd calendar schedule."; + }; + mode = mkOption { + type = types.enum ["normal" "deep"]; + default = "normal"; + description = "Semantic extraction depth."; + }; + codeOnly = mkOption { + type = types.bool; + default = false; + }; + noCluster = mkOption { + type = types.bool; + default = false; + }; + dedup = mkOption { + type = types.bool; + default = false; + }; + googleWorkspace = mkOption { + type = types.bool; + default = false; + }; + global = mkOption { + type = types.bool; + default = false; + }; + tag = mkOption { + type = types.nullOr types.str; + default = null; + }; + maxWorkers = mkOption { + type = nullablePositiveInt; + default = null; + }; + tokenBudget = mkOption { + type = nullablePositiveInt; + default = null; + }; + maxConcurrency = mkOption { + type = nullablePositiveInt; + default = null; + }; + apiTimeout = mkOption { + type = nullablePositiveNumber; + default = null; + }; + resolution = mkOption { + type = types.addCheck types.number (value: value > 0); + default = 1.0; + }; + excludeHubs = mkOption { + type = types.nullOr types.number; + default = null; + }; + excludes = mkOption { + type = types.listOf types.str; + default = []; + }; + timing = mkOption { + type = types.bool; + default = false; + }; + }; + }; + + llmOptions = {name, ...}: { + options = { + backend = mkOption { + type = types.nullOr types.str; + default = null; + example = "openai"; + description = "Built-in or custom Graphify backend name."; + }; + customProvider = mkOption { + type = types.bool; + default = false; + description = "Whether backend names an entry from providersFile instead of a built-in backend."; + }; + model = mkOption { + type = types.nullOr types.str; + default = null; + }; + baseUrl = mkOption { + type = types.nullOr types.str; + default = null; + description = "Built-in provider endpoint override."; + }; + apiKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Provider API key loaded through a systemd credential."; + }; + apiKeyEnvironmentVariable = mkOption { + type = types.nullOr (types.addCheck types.str (value: builtins.match "[A-Z_][A-Z0-9_]*" value != null)); + default = null; + example = "OPENAI_API_KEY"; + description = "Explicit key variable for a custom provider or non-default alias."; + }; + providersFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Custom providers.json mounted read-only at HOME/.graphify/providers.json."; + }; + }; + }; + + serverOptions = {name, ...}: { + options = { + enable = mkEnableOption "HTTP MCP serving for ${name}"; + host = mkOption { + type = types.str; + default = "127.0.0.1"; + }; + port = mkOption { + type = types.port; + default = 8080; + }; + path = mkOption { + type = types.str; + default = "/mcp"; + }; + jsonResponse = mkOption { + type = types.bool; + default = false; + }; + stateless = mkOption { + type = types.bool; + default = false; + }; + sessionTimeout = mkOption { + type = types.addCheck types.number (value: value >= 0); + default = 3600; + }; + apiKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "MCP bearer key loaded through a systemd credential."; + }; + unsafeAllowUnauthenticated = mkOption { + type = types.bool; + default = false; + description = "Explicitly allow an unauthenticated non-loopback listener."; + }; + openFirewall = mkOption { + type = types.bool; + default = false; + }; + }; + }; + + sinkOptions = sink: {name, ...}: { + options = { + enable = mkEnableOption "${sink} export for ${name}"; + uri = mkOption { + type = types.nullOr types.str; + default = null; + example = + if sink == "neo4j" + then "bolt://127.0.0.1:7687" + else "falkordb://127.0.0.1:6379"; + description = "Graph database URI passed to Graphify's push exporter."; + }; + user = mkOption { + type = types.nullOr types.str; + default = + if sink == "neo4j" + then "neo4j" + else null; + description = "Optional graph database user."; + }; + passwordFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Password loaded through a systemd credential and provider-specific environment variable."; + }; + onExtraction = mkOption { + type = types.bool; + default = true; + description = "Push after each successful extraction."; + }; + onCalendar = mkOption { + type = types.nullOr types.str; + default = null; + description = "Optional independent systemd calendar schedule."; + }; + }; + }; + + instanceOptions = {name, ...}: { + options = { + stateDirectory = mkOption { + type = types.str; + default = "/var/lib/graphify/${name}"; + description = "Mutable extraction state; graph.json is stored below graphify-out/."; + }; + + source = { + path = mkOption { + type = types.nullOr types.str; + default = null; + description = "Optional absolute runtime corpus path."; + }; + cargo = mkOption { + type = types.bool; + default = false; + }; + postgresql = mkOption { + type = types.submodule postgresOptions; + default = {}; + }; + }; + + extraction = mkOption { + type = types.submodule extractionOptions; + default = {}; + }; + llm = mkOption { + type = types.submodule llmOptions; + default = {}; + }; + + watch = { + enable = mkEnableOption "filesystem watching for ${name}"; + debounce = mkOption { + type = types.addCheck types.number (value: value > 0); + default = 3.0; + }; + }; + + server = mkOption { + type = types.submodule serverOptions; + default = {}; + }; + + exports = { + neo4j = mkOption { + type = types.submodule (sinkOptions "neo4j"); + default = {}; + }; + falkordb = mkOption { + type = types.submodule (sinkOptions "falkordb"); + default = {}; + }; + }; + + environment = mkOption { + type = types.attrsOf types.str; + default = {}; + description = "Additional non-secret Graphify/provider environment variables."; + }; + + environmentFiles = mkOption { + type = types.listOf types.path; + default = []; + description = "Systemd environment files for provider/AWS/runtime settings and secrets."; + }; + + runtimePackages = mkOption { + type = types.listOf types.package; + default = []; + example = lib.literalExpression "[ pkgs.claude-code pkgs.gws ]"; + description = "External executables added to service PATH, such as claude for claude-cli or gws for Google Workspace export."; + }; + }; + }; + + enabledInstances = cfg.instances; + + graphOut = instance: "${instance.stateDirectory}/graphify-out"; + graphPath = instance: "${graphOut instance}/graph.json"; + + llmKeyVariable = instance: + if instance.llm.apiKeyEnvironmentVariable != null + then instance.llm.apiKeyEnvironmentVariable + else if instance.llm.backend != null + then backendApiKeyVariables.${instance.llm.backend} or null + else null; + + baseEnvironment = instance: let + pg = instance.source.postgresql; + backend = instance.llm.backend; + baseUrlVariable = + if backend != null + then backendBaseUrlVariables.${backend} or null + else null; + in + { + HOME = instance.stateDirectory; + PYTHONDONTWRITEBYTECODE = "1"; + } + // optionalAttrs pg.enable { + PGHOST = pg.host; + PGPORT = toString pg.port; + PGDATABASE = pg.database; + PGUSER = pg.user; + PGSSLMODE = pg.sslMode; + } + // optionalAttrs (instance.llm.baseUrl != null && baseUrlVariable != null) { + ${baseUrlVariable} = instance.llm.baseUrl; + } + // instance.environment; + + extractionArguments = name: instance: + ["extract"] + ++ optional (instance.source.path != null) instance.source.path + ++ optional (instance.llm.backend != null) "--backend=${instance.llm.backend}" + ++ optional (instance.llm.model != null) "--model=${instance.llm.model}" + ++ optional (instance.extraction.mode == "deep") "--mode=deep" + ++ ["--out=${instance.stateDirectory}"] + ++ optional instance.extraction.noCluster "--no-cluster" + ++ optional instance.extraction.dedup "--dedup-llm" + ++ optional instance.extraction.codeOnly "--code-only" + ++ optional instance.extraction.googleWorkspace "--google-workspace" + ++ optional instance.extraction.global "--global" + ++ optional (instance.extraction.tag != null) "--as=${instance.extraction.tag}" + ++ optional (instance.extraction.maxWorkers != null) "--max-workers=${toString instance.extraction.maxWorkers}" + ++ optional (instance.extraction.tokenBudget != null) "--token-budget=${toString instance.extraction.tokenBudget}" + ++ optional (instance.extraction.maxConcurrency != null) "--max-concurrency=${toString instance.extraction.maxConcurrency}" + ++ optional (instance.extraction.apiTimeout != null) "--api-timeout=${toString instance.extraction.apiTimeout}" + ++ ["--resolution=${toString instance.extraction.resolution}"] + ++ optional (instance.extraction.excludeHubs != null) "--exclude-hubs=${toString instance.extraction.excludeHubs}" + ++ flatten (map (value: ["--exclude" value]) instance.extraction.excludes) + ++ optional instance.source.postgresql.enable "--postgres=" + ++ optional instance.source.cargo "--cargo" + ++ optional instance.extraction.timing "--timing"; + + credentialSetup = instance: let + keyVariable = llmKeyVariable instance; + in '' + ${optionalString (instance.source.postgresql.pgpassFile != null) '' + export PGPASSFILE="$CREDENTIALS_DIRECTORY/postgresql-pgpass" + ''} + ${optionalString (instance.llm.apiKeyFile != null && keyVariable != null) '' + export ${keyVariable}="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/llm-api-key")" + ''} + ''; + + extractionScript = name: instance: + pkgs.writeShellScript "graphify-${name}-extract" '' + set -eu + ${credentialSetup instance} + exec ${lib.getExe cfg.package} ${escapeShellArgs (extractionArguments name instance)} + ''; + + serverScript = name: instance: + pkgs.writeShellScript "graphify-${name}-mcp" '' + set -eu + ${optionalString (instance.server.apiKeyFile != null) '' + export GRAPHIFY_API_KEY="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/mcp-api-key")" + ''} + exec ${cfg.package}/bin/graphify-mcp ${escapeShellArgs ([ + (graphPath instance) + "--transport=http" + "--host=${instance.server.host}" + "--port=${toString instance.server.port}" + "--path=${instance.server.path}" + "--session-timeout=${toString instance.server.sessionTimeout}" + ] + ++ optional instance.server.jsonResponse "--json-response" + ++ optional instance.server.stateless "--stateless")} + ''; + + exportScript = name: instance: sink: sinkCfg: let + passwordVariable = + if sink == "neo4j" + then "NEO4J_PASSWORD" + else "FALKORDB_PASSWORD"; + in + pkgs.writeShellScript "graphify-${name}-${sink}-export" '' + set -eu + ${optionalString (sinkCfg.passwordFile != null) '' + export ${passwordVariable}="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/graph-database-password")" + ''} + exec ${lib.getExe cfg.package} ${escapeShellArgs ([ + "export" + sink + "--graph" + (graphPath instance) + "--push" + sinkCfg.uri + ] + ++ optional (sinkCfg.user != null) "--user=${sinkCfg.user}")} + ''; + + commonServiceConfig = instance: { + User = cfg.user; + Group = cfg.group; + UMask = "0077"; + NoNewPrivileges = true; + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = "read-only"; + ReadWritePaths = [instance.stateDirectory]; + ReadOnlyPaths = optional (instance.source.path != null) instance.source.path; + EnvironmentFile = map toString instance.environmentFiles; + }; + + instanceAssertions = concatLists (mapAttrsToList (name: instance: let + pg = instance.source.postgresql; + server = instance.server; + loopback = builtins.elem server.host ["127.0.0.1" "::1" "localhost"]; + keyVariable = llmKeyVariable instance; + sinkAssertions = concatLists (mapAttrsToList (sink: sinkCfg: [ + { + assertion = !sinkCfg.enable || sinkCfg.uri != null; + message = "services.graphify.instances.${name}.exports.${sink}.uri is required when the sink is enabled."; + } + { + assertion = !sinkCfg.enable || !sinkCfg.onExtraction || instance.extraction.enable; + message = "services.graphify.instances.${name}.exports.${sink}.onExtraction requires extraction.enable."; + } + { + assertion = sink != "neo4j" || !sinkCfg.enable || sinkCfg.passwordFile != null; + message = "services.graphify.instances.${name}.exports.neo4j.passwordFile is required by Neo4j push."; + } + ]) + instance.exports); + in + [ + { + assertion = !instance.extraction.enable || instance.source.path != null || pg.enable; + message = "services.graphify.instances.${name} needs source.path and/or source.postgresql.enable when extraction is enabled."; + } + { + assertion = !pg.enable || pg.database != null; + message = "services.graphify.instances.${name}.source.postgresql.database is required when PostgreSQL introspection is enabled."; + } + { + assertion = !instance.source.cargo || instance.source.path != null; + message = "services.graphify.instances.${name}.source.cargo requires source.path."; + } + { + assertion = !instance.watch.enable || instance.source.path != null; + message = "services.graphify.instances.${name}.watch.enable requires source.path."; + } + { + assertion = !instance.watch.enable || instance.extraction.enable; + message = "services.graphify.instances.${name}.watch.enable requires extraction.enable for the initial graph."; + } + { + assertion = lib.hasPrefix "/" instance.stateDirectory; + message = "services.graphify.instances.${name}.stateDirectory must be absolute."; + } + { + assertion = instance.source.path == null || lib.hasPrefix "/" instance.source.path; + message = "services.graphify.instances.${name}.source.path must be absolute."; + } + { + assertion = !(instance.environment ? GRAPHIFY_OUT) && !(instance.environment ? HOME); + message = "services.graphify.instances.${name}.environment must not override module-owned GRAPHIFY_OUT or HOME paths."; + } + { + assertion = !server.enable || lib.hasPrefix "/" server.path; + message = "services.graphify.instances.${name}.server.path must begin with '/'."; + } + { + assertion = !server.enable || loopback || server.apiKeyFile != null || server.unsafeAllowUnauthenticated; + message = "services.graphify.instances.${name} refuses unauthenticated non-loopback MCP; set server.apiKeyFile or unsafeAllowUnauthenticated."; + } + { + assertion = instance.llm.backend == null || instance.llm.customProvider || builtins.elem instance.llm.backend builtInBackends; + message = "services.graphify.instances.${name}.llm.backend is not built in; set llm.customProvider for providersFile entries."; + } + { + assertion = !instance.llm.customProvider || instance.llm.providersFile != null; + message = "services.graphify.instances.${name}.llm.customProvider requires llm.providersFile."; + } + { + assertion = instance.llm.apiKeyFile == null || keyVariable != null; + message = "services.graphify.instances.${name}.llm.apiKeyFile requires a known backend or apiKeyEnvironmentVariable."; + } + ] + ++ sinkAssertions) + enabledInstances); +in { + options.services.graphify = { + enable = mkEnableOption "Graphify extraction and MCP services"; + + package = mkOption { + type = types.nullOr types.package; + default = graphifyPackage; + defaultText = lib.literalExpression "graphify.packages..full"; + description = "Graphify package. The upstream module supplies the full runtime package."; + }; + + user = mkOption { + type = types.str; + default = "graphify"; + }; + group = mkOption { + type = types.str; + default = "graphify"; + }; + instances = mkOption { + type = types.attrsOf (types.submodule instanceOptions); + default = {}; + description = "Named Graphify graphs with independent sources, state, schedules, and MCP listeners."; + }; + }; + + config = mkIf cfg.enable (mkMerge [ + { + assertions = + [ + { + assertion = cfg.package != null; + message = "services.graphify.package must be set when Graphify is enabled."; + } + { + assertion = cfg.instances != {}; + message = "services.graphify.instances must contain at least one instance."; + } + ] + ++ instanceAssertions; + + users.groups.${cfg.group} = {}; + users.users.${cfg.user} = { + isSystemUser = true; + group = cfg.group; + home = "/var/lib/graphify"; + createHome = true; + }; + + systemd.tmpfiles.rules = flatten (mapAttrsToList (_: instance: [ + "d ${instance.stateDirectory} 0750 ${cfg.user} ${cfg.group} - -" + "d ${instance.stateDirectory}/.graphify 0750 ${cfg.user} ${cfg.group} - -" + ]) + enabledInstances); + + networking.firewall.allowedTCPPorts = + unique (mapAttrsToList (_: instance: instance.server.port) + (filterAttrs (_: instance: instance.server.enable && instance.server.openFirewall) enabledInstances)); + } + + { + systemd.services = mkMerge (mapAttrsToList (name: instance: let + extractUnit = "graphify-${name}-extract"; + pgService = instance.source.postgresql.systemdService; + providerBind = + optional (instance.llm.providersFile != null) + "${toString instance.llm.providersFile}:${instance.stateDirectory}/.graphify/providers.json"; + in + mkMerge ([ + (mkIf instance.extraction.enable { + ${extractUnit} = { + description = "Extract Graphify graph ${name}"; + wantedBy = optional instance.extraction.startAtBoot "multi-user.target"; + after = optional (pgService != null) pgService; + requires = optional (pgService != null) pgService; + environment = baseEnvironment instance; + path = instance.runtimePackages; + unitConfig.OnSuccess = + mapAttrsToList + (sink: _: "graphify-${name}-${sink}.service") + (filterAttrs (_: sinkCfg: sinkCfg.enable && sinkCfg.onExtraction) instance.exports); + serviceConfig = + commonServiceConfig instance + // { + Type = "oneshot"; + ExecStart = extractionScript name instance; + LoadCredential = + optional (instance.source.postgresql.pgpassFile != null) "postgresql-pgpass:${toString instance.source.postgresql.pgpassFile}" + ++ optional (instance.llm.apiKeyFile != null) "llm-api-key:${toString instance.llm.apiKeyFile}"; + BindReadOnlyPaths = providerBind; + }; + }; + }) + (mkIf instance.watch.enable { + "graphify-${name}-watch" = { + description = "Watch Graphify corpus ${name}"; + wantedBy = ["multi-user.target"]; + after = ["${extractUnit}.service"]; + environment = baseEnvironment instance // {GRAPHIFY_OUT = graphOut instance;}; + path = instance.runtimePackages; + serviceConfig = + commonServiceConfig instance + // { + ExecStart = "${lib.getExe cfg.package} watch ${lib.escapeShellArg instance.source.path} --debounce ${toString instance.watch.debounce}"; + Restart = "on-failure"; + RestartSec = "5s"; + }; + }; + }) + (mkIf instance.server.enable { + "graphify-${name}" = { + description = "Graphify MCP server ${name}"; + wantedBy = ["multi-user.target"]; + after = optional instance.extraction.enable "${extractUnit}.service"; + environment = baseEnvironment instance; + path = instance.runtimePackages; + unitConfig.ConditionPathExists = graphPath instance; + serviceConfig = + commonServiceConfig instance + // { + ExecStart = serverScript name instance; + Restart = "on-failure"; + RestartSec = "5s"; + LoadCredential = optional (instance.server.apiKeyFile != null) "mcp-api-key:${toString instance.server.apiKeyFile}"; + }; + }; + }) + ] + ++ mapAttrsToList (sink: sinkCfg: + mkIf sinkCfg.enable { + "graphify-${name}-${sink}" = { + description = "Push Graphify graph ${name} to ${sink}"; + after = optional instance.extraction.enable "${extractUnit}.service"; + environment = baseEnvironment instance; + path = instance.runtimePackages; + unitConfig.ConditionPathExists = graphPath instance; + serviceConfig = + commonServiceConfig instance + // { + Type = "oneshot"; + ExecStart = exportScript name instance sink sinkCfg; + LoadCredential = optional (sinkCfg.passwordFile != null) "graph-database-password:${toString sinkCfg.passwordFile}"; + }; + }; + }) + instance.exports)) + enabledInstances); + + systemd.timers = mkMerge ([ + (mapAttrs' (name: instance: + nameValuePair "graphify-${name}-extract" { + wantedBy = ["timers.target"]; + timerConfig = { + OnCalendar = instance.extraction.onCalendar; + Persistent = true; + Unit = "graphify-${name}-extract.service"; + }; + }) (filterAttrs (_: instance: instance.extraction.enable && instance.extraction.onCalendar != null) enabledInstances)) + ] + ++ flatten (mapAttrsToList (name: instance: + mapAttrsToList (sink: sinkCfg: + mkIf (sinkCfg.enable && sinkCfg.onCalendar != null) { + "graphify-${name}-${sink}" = { + wantedBy = ["timers.target"]; + timerConfig = { + OnCalendar = sinkCfg.onCalendar; + Persistent = true; + Unit = "graphify-${name}-${sink}.service"; + }; + }; + }) + instance.exports) + enabledInstances)); + } + ]); +} diff --git a/nix/pre-commit.nix b/nix/pre-commit.nix new file mode 100644 index 000000000..c6d6d3b4a --- /dev/null +++ b/nix/pre-commit.nix @@ -0,0 +1,28 @@ +{ + pkgs, + treefmtWrapper, +}: { + treefmt = { + enable = true; + name = "treefmt"; + entry = "${treefmtWrapper}/bin/treefmt --fail-on-change"; + pass_filenames = false; + }; + + nix-flake-check = { + enable = true; + name = "nix flake check"; + entry = "nix --extra-experimental-features 'nix-command flakes' flake check --cores 0 --max-jobs auto --no-update-lock-file"; + extraPackages = [pkgs.nix]; + pass_filenames = false; + stages = ["manual"]; + }; + + uv-ruff-format = { + enable = true; + name = "uv ruff format"; + entry = "uv run ruff format --check ."; + extraPackages = [pkgs.uv]; + pass_filenames = false; + }; +} diff --git a/pyproject.toml b/pyproject.toml index b8ea4ad31..e4784f00b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ pascal = ["tree-sitter-pascal"] # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "psycopg[binary]", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/simit.toml b/simit.toml new file mode 100644 index 000000000..98687d863 --- /dev/null +++ b/simit.toml @@ -0,0 +1,12 @@ +[flake] +mode = "custom" +scope = "hooks-only" +components = ["treefmt", "nix-flake-check", "uv-ruff-format"] + +[flake.expected_outputs] +checks = ["pytest"] + +[ci] +runtime = "nix" +runner = "ubuntu-latest" +components = ["flake-wiring", "flake-evaluation", "checks"] diff --git a/tests/test_backend_extras.py b/tests/test_backend_extras.py index f513c57dc..4da79a007 100644 --- a/tests/test_backend_extras.py +++ b/tests/test_backend_extras.py @@ -34,6 +34,12 @@ def test_anthropic_in_all_extra(): assert any("anthropic" in dep for dep in extras["all"]), "[all] must include anthropic" +def test_postgres_in_all_extra(): + extras = _extras() + assert "psycopg[binary]" in extras["postgres"] + assert "psycopg[binary]" in extras["all"], "[all] must include PostgreSQL support" + + def test_backend_pkg_hint_points_at_uv_tool_and_extra(): msg = _backend_pkg_hint("anthropic", "anthropic") assert "uv tool install" in msg diff --git a/tests/test_extract.py b/tests/test_extract.py index 98b73edb9..f9f7a5ce7 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -382,7 +382,7 @@ def test_collect_files_from_dir(): def test_collect_files_skips_hidden(): files = collect_files(FIXTURES) for f in files: - assert not any(part.startswith(".") for part in f.parts) + assert not any(part.startswith(".") for part in f.relative_to(FIXTURES).parts) def test_collect_files_follows_symlinked_directory(tmp_path): @@ -445,7 +445,7 @@ def _legacy_collect_files(target, *, root=None): results.extend( p for p in target.rglob(f"*{ext}") if p.suffix == ext - and not any(_is_noise_dir(part) for part in p.parts) + and not any(_is_noise_dir(part) for part in p.relative_to(target).parts) and not (patterns and _is_ignored(p, ignore_root, patterns)) ) return sorted(results) diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index 4133002c0..dc02a65b3 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -1,13 +1,84 @@ """Tests for `graphify extract` CLI dispatch path in graphify.__main__.""" from __future__ import annotations -import os +import json import pytest import graphify.__main__ as mainmod +def _postgres_result(): + """Minimal valid database extraction returned by the introspection stub.""" + source = "postgresql:/db.example.test/app" + return { + "nodes": [ + { + "id": "postgresql:app:public.widgets", + "label": "public.widgets", + "type": "table", + "file_type": "code", + "source_file": source, + "source_location": f"{source}:1", + "confidence": "EXTRACTED", + } + ], + "edges": [], + } + + +@pytest.mark.parametrize("with_path", [False, True], ids=["postgres-only", "path-and-postgres"]) +def test_extract_postgres_reaches_introspection_and_writes_output( + monkeypatch, tmp_path, with_path +): + """PostgreSQL-only mode must initialize the empty detection result. + + The no-path form previously skipped ``detect()`` as intended, then crashed + with ``UnboundLocalError`` when common reporting code read ``detection``. + The path form is covered alongside it to preserve combined source + schema + extraction while fixing the database-only branch. + """ + out_dir = tmp_path / "out" + argv = ["graphify", "extract"] + if with_path: + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def application_entry():\n return 1\n") + argv.append(str(corpus)) + argv.extend( + [ + "--postgres", + "postgresql://reader:secret@db.example.test/app", + "--out", + str(out_dir), + "--no-cluster", + ] + ) + + calls = [] + + def _introspect(dsn): + calls.append(dsn) + return _postgres_result() + + monkeypatch.setattr("graphify.pg_introspect.introspect_postgres", _introspect) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", argv) + + with pytest.raises(SystemExit) as exc_info: + mainmod.main() + + assert exc_info.value.code == 0 + assert calls == ["postgresql://reader:secret@db.example.test/app"] + + graph_path = out_dir / "graphify-out" / "graph.json" + graph = json.loads(graph_path.read_text(encoding="utf-8")) + labels = {node.get("label") for node in graph["nodes"]} + assert "public.widgets" in labels + if with_path: + assert any(str(label).startswith("application_entry") for label in labels) + + def _make_corpus(tmp_path): """Minimal corpus: one Go code file + one Markdown doc. @@ -104,15 +175,6 @@ def _one_chunk_succeeded(paths, **kwargs): monkeypatch.setattr( "graphify.llm.extract_corpus_parallel", _one_chunk_succeeded ) - cache_call = {} - - def _capture_semantic_cache(*args, **kwargs): - cache_call.update(kwargs) - return 0 - - monkeypatch.setattr( - "graphify.cache.save_semantic_cache", _capture_semantic_cache - ) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -132,450 +194,6 @@ def _capture_semantic_cache(*args, **kwargs): assert (out_dir / "graphify-out" / "graph.json").exists(), ( "graph.json must be written on the happy path" ) - assert { - str(path) for path in cache_call["allowed_source_files"] - } == {str(corpus / "README.md")} - - -def test_incremental_partial_run_preserves_untouched_semantic_hash( - monkeypatch, tmp_path -): - """#1948 caller-side guard: an incremental run that only re-dispatches the - CHANGED subset must not blank semantic_hash for live-but-untouched files. - - clear_semantic must be derived from what was actually SENT to the backend - this run (semantic_files), not from the full live corpus (files_by_type): - with the latter, every unchanged doc lands in the clear set on every - incremental run, so the very next run re-extracts the whole corpus, - forever.""" - import json - - corpus = _make_corpus(tmp_path) # main.go + README.md - (corpus / "OTHER.md").write_text("# Other\nAn independent second doc.\n") - out_dir = tmp_path / "out" - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - - dispatched: list[list[str]] = [] - - def _stamp_everything_sent(paths, **kwargs): - sent = sorted(os.path.relpath(str(p), str(corpus)) for p in paths) - dispatched.append(sent) - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - return { - "nodes": [{"id": f"n-{rel}", "source_file": rel, - "file_type": "document"} for rel in sent], - "edges": [], - "hyperedges": [], - "input_tokens": 10, - "output_tokens": 5, - } - - monkeypatch.setattr( - "graphify.llm.extract_corpus_parallel", _stamp_everything_sent - ) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - def _run_extract(): - monkeypatch.setattr( - mainmod.sys, "argv", - ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster", "--out", str(out_dir)], - ) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - # Run 1: full scan — both docs dispatched and stamped. - _run_extract() - manifest_path = out_dir / "graphify-out" / "manifest.json" - m1 = json.loads(manifest_path.read_text()) - assert m1["README.md"].get("semantic_hash") - assert m1["OTHER.md"].get("semantic_hash") - - # Run 2: only README.md changes → the incremental gate dispatches it alone. - (corpus / "README.md").write_text("# Notes\nChanged content, new hash.\n") - _run_extract() - assert dispatched[-1] == ["README.md"], ( - f"run 2 should dispatch only the changed doc, got {dispatched[-1]}" - ) - m2 = json.loads(manifest_path.read_text()) - assert m2["README.md"].get("semantic_hash") - # The heart of the guard: an untouched, never-dispatched live doc keeps - # its stamp across a partial incremental run. - assert m2["OTHER.md"].get("semantic_hash"), ( - "untouched doc's semantic_hash was blanked by a partial incremental " - "run — clear_semantic was derived from the full live corpus instead " - "of the dispatched subset (#1948)" - ) - - -def test_truncated_doc_semantic_hash_is_cleared_for_requeue(monkeypatch, tmp_path): - """#1948 x #1950 interaction: a doc stamped complete on a prior run that - TRUNCATES (partial) this run must have its stale semantic_hash cleared, so - detect_incremental re-queues it — not inherit the old hash and look - unchanged. Partial files are dropped by _stamped_manifest_files, so they - land in clear_semantic (dispatched-but-not-stamped).""" - import json - - corpus = _make_corpus(tmp_path) # main.go + README.md - out_dir = tmp_path / "out" - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - partial_run = {"on": False} - - def _extract(paths, **kwargs): - rels = sorted(os.path.relpath(str(p), str(corpus)) for p in paths) - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - node = {"id": "n-readme", "source_file": "README.md", "file_type": "document"} - if partial_run["on"] and "README.md" in rels: - node["_partial"] = True # this run truncated README.md - return {"nodes": [node] if "README.md" in rels else [], - "edges": [], "hyperedges": [], "input_tokens": 10, "output_tokens": 5} - - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _extract) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - def _run(): - monkeypatch.setattr(mainmod.sys, "argv", - ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster", "--out", str(out_dir)]) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0) - - manifest_path = out_dir / "graphify-out" / "manifest.json" - _run() # run 1: complete - assert json.loads(manifest_path.read_text())["README.md"].get("semantic_hash") - - # run 2: README.md changes and truncates (partial) this time. - (corpus / "README.md").write_text("# Notes\nNew, longer content that truncated.\n") - partial_run["on"] = True - _run() - m2 = json.loads(manifest_path.read_text()) - assert not m2.get("README.md", {}).get("semantic_hash"), ( - "a truncated doc's stale semantic_hash must be cleared so it is " - "re-queued next run (#1948 x #1950)" - ) - - -def test_manifest_stamps_freshly_extracted_semantic_docs(monkeypatch, tmp_path): - """#1897: fresh extraction returns nodes with ROOT-RELATIVE source_file, - while the #933 manifest filter compared them against detect()'s ABSOLUTE - paths — so `f in _sem_extracted` was always False and every freshly - extracted doc was dropped from the manifest (only code/zero-node files - survived). Both sides must be resolved against the scan root; a genuinely - omitted doc (zero nodes) must still stay unstamped (#933 is intentional).""" - import json - - corpus = _make_corpus(tmp_path) # main.go + README.md - (corpus / "OMITTED.md").write_text("# never extracted\n") - out_dir = tmp_path / "out" - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - - def _fresh_relative(paths, **kwargs): - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - # Root-relative source_file, exactly what a fresh extraction produces. - # OMITTED.md gets no nodes/edges — the model skipped it. - return { - "nodes": [{"id": "readme", "source_file": "README.md", - "file_type": "document"}], - "edges": [], - "hyperedges": [], - "input_tokens": 10, - "output_tokens": 5, - } - - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _fresh_relative) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr( - mainmod.sys, "argv", - ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster", "--out", str(out_dir)], - ) - - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - manifest_path = out_dir / "graphify-out" / "manifest.json" - assert manifest_path.exists() - manifest = json.loads(manifest_path.read_text()) - - assert "README.md" in manifest, ( - f"freshly-extracted doc missing from manifest (#1897): {sorted(manifest)}" - ) - assert manifest["README.md"].get("semantic_hash"), ( - "freshly-extracted doc must carry a non-empty semantic_hash" - ) - # Code files are always stamped. - assert manifest.get("main.go", {}).get("semantic_hash") - # The zero-node doc stays unstamped so detect_incremental re-queues it (#933). - assert "OMITTED.md" not in manifest, ( - "zero-node doc must not be stamped in the manifest" - ) - - -def test_stamped_manifest_files_normalizes_both_sides(tmp_path): - """Unit test for the #1897 helper: relative (fresh) and absolute (cache-hit) - source_file values must both match detect()'s absolute file lists; docs with - no output are filtered; code files pass through untouched.""" - from graphify.cli import _stamped_manifest_files - - fresh_doc = tmp_path / "fresh.md"; fresh_doc.write_text("# fresh") - cached_doc = tmp_path / "cached.md"; cached_doc.write_text("# cached") - omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") - code = tmp_path / "app.py"; code.write_text("x = 1") - - files_by_type = { - "code": [str(code)], - "document": [str(fresh_doc), str(cached_doc), str(omitted_doc)], - } - sem_result = { - # fresh extraction: root-relative source_file - "nodes": [{"id": "n1", "source_file": "fresh.md"}], - # cache replay: absolute source_file (edge-only coverage counts too) - "edges": [{"source": "a", "target": "b", "source_file": str(cached_doc)}], - } - - out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) - assert out["code"] == [str(code)] - assert out["document"] == [str(fresh_doc), str(cached_doc)] - - -def test_stamped_manifest_files_counts_hyperedge_only_docs(tmp_path): - """#1920: a doc whose only chunk output is a hyperedge (3+ nodes sharing a - concept) is valid output — the semantic cache persists it per source_file — - so it must be stamped. Before the fix the stamping loop only inspected - ``nodes``/``edges``, leaving such a doc unstamped and re-queued forever.""" - from graphify.cli import _stamped_manifest_files - - hyper_doc = tmp_path / "hyper.md"; hyper_doc.write_text("# hyper") - omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") - - files_by_type = {"document": [str(hyper_doc), str(omitted_doc)]} - sem_result = { - "nodes": [], - "edges": [], - "hyperedges": [ - {"id": "h1", "label": "L", "nodes": ["a", "b", "c"], - "relation": "participate_in", "source_file": "hyper.md"}, - ], - } - - out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) - assert str(hyper_doc) in out["document"], ( - "a hyperedge-only doc must be stamped (#1920)" - ) - # A doc with no output at all still stays unstamped (#933). - assert str(omitted_doc) not in out["document"] - - -def test_manifest_stamps_hyperedge_only_docs(monkeypatch, tmp_path): - """#1920 end-to-end: a fresh extraction whose only output for a doc is a - hyperedge stamps that doc's semantic_hash, so it is not re-dispatched.""" - import json - - corpus = _make_corpus(tmp_path) # main.go + README.md - out_dir = tmp_path / "out" - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - - def _hyperedge_only(paths, **kwargs): - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - return { - "nodes": [], - "edges": [], - "hyperedges": [{"id": "h1", "label": "Shared", "nodes": ["a", "b", "c"], - "relation": "participate_in", "source_file": "README.md"}], - "input_tokens": 10, - "output_tokens": 5, - } - - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _hyperedge_only) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - monkeypatch.setattr( - mainmod.sys, "argv", - ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster", "--out", str(out_dir)], - ) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - manifest = json.loads((out_dir / "graphify-out" / "manifest.json").read_text()) - assert manifest.get("README.md", {}).get("semantic_hash"), ( - f"hyperedge-only doc must be stamped (#1920): {sorted(manifest)}" - ) - - -# --- #1894: --force and deep-mode dispatch over a warm cache ----------------- - -def _recording_extractor(calls): - """extract_corpus_parallel stand-in that records each dispatch.""" - def _extract(paths, **kwargs): - calls.append({"paths": [str(p) for p in paths], "kwargs": kwargs}) - on_chunk = kwargs.get("on_chunk_done") - if on_chunk: - on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) - return { - "nodes": [{"id": "readme", "source_file": "README.md", - "file_type": "document"}], - "edges": [], - "hyperedges": [], - "input_tokens": 10, - "output_tokens": 5, - } - return _extract - - -def _run_extract(monkeypatch, argv): - monkeypatch.setattr(mainmod.sys, "argv", argv) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - -def test_extract_mode_deep_dispatches_over_warm_cache(monkeypatch, tmp_path): - """#1894 repro: over a warm manifest + warm standard semantic cache, - `extract --mode deep` was a silent no-op — the incremental gate dispatched - zero files before the cache was ever consulted, and the cache key ignored - mode anyway. Deep must re-dispatch on the first deep run (deep namespace - cold) and be served from cache/semantic-deep/ on the second.""" - corpus = _make_corpus(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) - calls: list[dict] = [] - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", - _recording_extractor(calls)) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - # No --out: the default layout (graphify-out/ beside the sources) keeps the - # CLI-level cache write's root anchored at the corpus, so the stub's - # root-relative source_file resolves (real runs also checkpoint per chunk - # inside llm.extract_corpus_parallel, which this stub replaces). - base = ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster"] - - # Run 1: cold standard extraction — warms manifest + plain semantic cache. - _run_extract(monkeypatch, base) - assert len(calls) == 1 - - # Sanity: a warm standard re-run dispatches nothing (expected behavior). - _run_extract(monkeypatch, base) - assert len(calls) == 1 - - # The repro: warm tree + --mode deep MUST dispatch. - _run_extract(monkeypatch, base + ["--mode", "deep"]) - assert len(calls) == 2, ( - "--mode deep over a warm cache must re-dispatch (#1894)" - ) - assert calls[1]["paths"] == [str(corpus / "README.md")] - assert calls[1]["kwargs"].get("deep_mode") is True - - # Second deep run: served from the (now warm) deep namespace, no dispatch. - _run_extract(monkeypatch, base + ["--mode", "deep"]) - assert len(calls) == 2, ( - "second deep run must be served from cache/semantic-deep/" - ) - # The deep entry landed in its own namespace, not cache/semantic/. Entries are - # nested under a p{prompt-fingerprint}/ subdir (#1939), hence the recursive glob. - assert any((corpus / "graphify-out" / "cache" / "semantic-deep").glob("**/*.json")) - - -def test_extract_force_flag_redispatches_and_stamps_manifest(monkeypatch, tmp_path): - """extract accepts --force: a warm tree re-dispatches every semantic file - (cache read skipped, incremental gate off) and the manifest is still - stamped afterward (#1897-compatible full coverage).""" - import json - - corpus = _make_corpus(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) - calls: list[dict] = [] - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", - _recording_extractor(calls)) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - base = ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster"] - - _run_extract(monkeypatch, base) - assert len(calls) == 1 - _run_extract(monkeypatch, base) # warm: no dispatch - assert len(calls) == 1 - - _run_extract(monkeypatch, base + ["--force"]) - assert len(calls) == 2, ( - "--force over a warm tree must re-dispatch every semantic file" - ) - assert calls[1]["paths"] == [str(corpus / "README.md")] - - # The forced run still wrote the semantic cache and stamped the manifest. - # Entries nest under a p{prompt-fingerprint}/ subdir (#1939). - assert any((corpus / "graphify-out" / "cache" / "semantic").glob("**/*.json")) - manifest = json.loads( - (corpus / "graphify-out" / "manifest.json").read_text() - ) - assert manifest.get("README.md", {}).get("semantic_hash"), ( - "forced re-dispatch must still stamp the manifest" - ) - assert manifest.get("main.go", {}).get("semantic_hash") - - -def test_extract_graphify_force_env_redispatches(monkeypatch, tmp_path): - """GRAPHIFY_FORCE=1 behaves like --force (env parity with `update`).""" - corpus = _make_corpus(tmp_path) - monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") - monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) - calls: list[dict] = [] - monkeypatch.setattr("graphify.llm.extract_corpus_parallel", - _recording_extractor(calls)) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - base = ["graphify", "extract", str(corpus), "--backend", "claude", - "--no-cluster"] - - _run_extract(monkeypatch, base) - assert len(calls) == 1 - _run_extract(monkeypatch, base) # warm: no dispatch - assert len(calls) == 1 - - monkeypatch.setenv("GRAPHIFY_FORCE", "1") - _run_extract(monkeypatch, base) - assert len(calls) == 2, "GRAPHIFY_FORCE=1 must force a re-dispatch" - - -def test_cache_check_mode_deep_reads_deep_namespace(monkeypatch, tmp_path, capsys): - """cache-check --mode deep consults cache/semantic-deep/; without the flag - it keeps reading cache/semantic/ (deep entries are invisible to it).""" - from graphify.cache import save_semantic_cache - - doc = tmp_path / "doc.md" - doc.write_text("# Doc\n") - save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], - root=tmp_path, mode="deep") - files_from = tmp_path / "files.txt" - files_from.write_text(str(doc) + "\n") - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - _run_extract(monkeypatch, ["graphify", "cache-check", str(files_from), - "--root", str(tmp_path)]) - assert "Cache: 0 hit, 1 miss" in capsys.readouterr().out - - _run_extract(monkeypatch, ["graphify", "cache-check", str(files_from), - "--root", str(tmp_path), "--mode", "deep"]) - assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out def _code_only_corpus(tmp_path): @@ -627,73 +245,6 @@ def test_extract_codeonly_succeeds_without_api_key(monkeypatch, tmp_path): assert len(json.loads(graph.read_text()).get("nodes", [])) > 0 -def test_missing_manifest_code_only_preserves_semantic_layer(monkeypatch, tmp_path): - """#1925: `graphify extract --code-only` with a MISSING manifest.json must - not degrade to a full scan that discards the committed semantic layer. An - existing graph.json is a sufficient incremental baseline, so doc/paper/image - nodes (excluded by --code-only, not deleted) are preserved; a genuinely - deleted source is still evicted (#1909 semantics retained).""" - import json - - corpus = tmp_path / "proj"; corpus.mkdir() - (corpus / "keep.py").write_text("def keep():\n return 1\n") - (corpus / "README.md").write_text("# Notes\nCurated docs.\n") - out_dir = tmp_path / "out" - graphify_out = out_dir / "graphify-out" - _clear_backend_keys(monkeypatch) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - def _sem_doc_count(g): - return sum(1 for n in g["nodes"] if n.get("source_file") == "README.md") - - # 1) seed a code-only graph - _run_extract(monkeypatch, ["graphify", "extract", str(corpus), - "--code-only", "--out", str(out_dir)]) - graph_path = graphify_out / "graph.json" - graph = json.loads(graph_path.read_text()) - - # 2) inject a committed semantic layer for README.md (nodes + edge + hyperedge) - graph["nodes"].append({"id": "doc_readme_a", "label": "Concept A", - "source_file": "README.md", "file_type": "document"}) - graph["nodes"].append({"id": "doc_readme_b", "label": "Concept B", - "source_file": "README.md", "file_type": "document"}) - graph.setdefault("edges", []).append( - {"source": "doc_readme_a", "target": "doc_readme_b", - "relation": "relates_to", "source_file": "README.md"}) - graph.setdefault("hyperedges", []).append( - {"id": "h1", "label": "Shared", "nodes": ["doc_readme_a", "doc_readme_b"], - "relation": "participate_in", "source_file": "README.md"}) - graph_path.write_text(json.dumps(graph)) - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": 1})) - - # 3) manifest goes missing (fresh clone / deliberately untracked) - (graphify_out / "manifest.json").unlink() - - # 4) re-run the SAME code-only extract - _run_extract(monkeypatch, ["graphify", "extract", str(corpus), - "--code-only", "--out", str(out_dir)]) - after = json.loads(graph_path.read_text()) - assert _sem_doc_count(after) >= 2, ( - "committed semantic doc nodes must survive a missing-manifest " - f"--code-only rebuild (#1925); got {_sem_doc_count(after)}" - ) - assert any(h.get("id") == "h1" for h in after.get("hyperedges", [])), ( - "committed hyperedge must survive the rebuild" - ) - assert any("keep" in n["id"] for n in after["nodes"]), "code nodes intact" - - # 5) a genuine deletion still evicts the doc's semantic nodes - (corpus / "README.md").unlink() - (graphify_out / "manifest.json").unlink(missing_ok=True) - _run_extract(monkeypatch, ["graphify", "extract", str(corpus), - "--code-only", "--out", str(out_dir)]) - gone = json.loads(graph_path.read_text()) - assert _sem_doc_count(gone) == 0, ( - "a genuinely deleted doc must still be evicted (#1909 semantics preserved)" - ) - - def test_extract_out_keeps_project_root_clean(monkeypatch, tmp_path): """`extract --out DIR` routes every artifact to DIR/graphify-out/ and the scanned project must not grow a graphify-out/ (or anything else) beside @@ -787,197 +338,3 @@ def test_extract_timing_flag_emits_stage_timings(monkeypatch, tmp_path, capsys): mainmod.main() assert exc2.value.code == 0 assert "graphify timing" not in capsys.readouterr().err - - -# --------------------------------------------------------------------------- -# #1909: a newly-excluded file's nodes must be pruned from graph.json on the -# next incremental extract even when the manifest never listed the file (the -# pre-#1897 state every 0.9.16 graph is in), so the manifest-diff prune set -# (`manifest - corpus`) can never see it. -# --------------------------------------------------------------------------- - -def _two_file_corpus(tmp_path): - project = tmp_path / "project" - project.mkdir() - (project / "x.py").write_text( - "def secret_helper():\n return 42\n\n" - "def secret_caller():\n return secret_helper()\n" - ) - (project / "keep.py").write_text( - "def kept():\n return still_here()\n\n" - "def still_here():\n return 1\n" - ) - return project - - -def _node_sources(graph_path): - import json - data = json.loads(graph_path.read_text(encoding="utf-8")) - return {n.get("source_file", "") for n in data.get("nodes", [])} - - -def _run_extract(monkeypatch, argv): - monkeypatch.setattr(mainmod.sys, "argv", argv) - try: - mainmod.main() - except SystemExit as exc: - assert exc.code in (None, 0), f"unexpected exit code {exc.code}" - - -def test_incremental_extract_prunes_newly_excluded_file_not_in_manifest( - monkeypatch, tmp_path -): - """Seed a graph with nodes for x.py, drop x.py from the manifest (pre-#1897 - manifests never listed excluded/omitted files), exclude x.py via - .graphifyignore, re-run extract: x.py's nodes must be gone even though it - was never on the deleted list.""" - import json - project = _two_file_corpus(tmp_path) - out_dir = tmp_path / "out" - _clear_backend_keys(monkeypatch) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - graph_path = out_dir / "graphify-out" / "graph.json" - manifest_path = out_dir / "graphify-out" / "manifest.json" - assert any("x.py" in s for s in _node_sources(graph_path)), ( - "seed extract must produce nodes for x.py" - ) - - # Simulate the pre-#1897 manifest state: x.py was never manifest-listed, - # so `manifest - corpus` can never flag it. - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest = {k: v for k, v in manifest.items() if "x.py" not in k} - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - - (project / ".graphifyignore").write_text("x.py\n") - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - - sources = _node_sources(graph_path) - assert not any("x.py" in s for s in sources), ( - f"newly-excluded x.py must be pruned from graph.json, still see {sources}" - ) - assert any("keep.py" in s for s in sources), ( - "unchanged keep.py nodes must survive the incremental merge" - ) - # x.py exists on disk, is excluded, and must not creep into the manifest. - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - assert not any("x.py" in k for k in manifest), ( - f"excluded x.py must not be (re)listed in the manifest: {set(manifest)}" - ) - - -def test_incremental_extract_prunes_excluded_file_listed_in_manifest( - monkeypatch, tmp_path -): - """Post-#1897 state: the excluded file IS manifest-listed. It must be - pruned from graph.json AND dropped from the manifest (#1908), and stay - settled on a further run.""" - import json - project = _two_file_corpus(tmp_path) - out_dir = tmp_path / "out" - _clear_backend_keys(monkeypatch) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - graph_path = out_dir / "graphify-out" / "graph.json" - manifest_path = out_dir / "graphify-out" / "manifest.json" - assert any("x.py" in k for k in json.loads(manifest_path.read_text())) - - (project / ".graphifyignore").write_text("x.py\n") - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - - sources = _node_sources(graph_path) - assert not any("x.py" in s for s in sources) - assert any("keep.py" in s for s in sources) - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - assert not any("x.py" in k for k in manifest), ( - "excluded-but-alive manifest row must be pruned (#1908)" - ) - - # Steady state: a third run neither resurrects x.py nor loses keep.py. - _run_extract( - monkeypatch, - ["graphify", "extract", str(project), "--out", str(out_dir)], - ) - sources = _node_sources(graph_path) - assert not any("x.py" in s for s in sources) - assert any("keep.py" in s for s in sources) - - -def test_no_cluster_incremental_prunes_newly_excluded_file( - monkeypatch, tmp_path, capsys -): - """--no-cluster's exclusion-only early exit must still scrub the excluded - file's nodes from the raw graph.json (that path never runs build_merge), - and must not report the alive file as deleted.""" - import json - project = _two_file_corpus(tmp_path) - out_dir = tmp_path / "out" - _clear_backend_keys(monkeypatch) - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - monkeypatch.setattr( - mainmod.sys, "argv", - ["graphify", "extract", str(project), "--no-cluster", "--out", str(out_dir)], - ) - with pytest.raises(SystemExit) as exc: - mainmod.main() - assert exc.value.code == 0 - graph_path = out_dir / "graphify-out" / "graph.json" - assert any("x.py" in s for s in _node_sources(graph_path)) - capsys.readouterr() - - (project / ".graphifyignore").write_text("x.py\n") - with pytest.raises(SystemExit) as exc: - mainmod.main() - assert exc.value.code == 0 - out_text = capsys.readouterr().out - assert "1 deleted" not in out_text, ( - "excluded-but-alive file must not be reported as deleted" - ) - - sources = _node_sources(graph_path) - assert not any("x.py" in s for s in sources), ( - f"--no-cluster early exit must prune excluded sources, still see {sources}" - ) - assert any("keep.py" in s for s in sources) - - -def test_cache_check_prompt_file_scopes_hits_to_that_prompt(monkeypatch, tmp_path, capsys): - """#1939: cache-check --prompt-file only counts entries produced by that same - extraction prompt, so an upgraded prompt reports a miss (re-extract) rather - than replaying the older vintage.""" - from graphify.cache import save_semantic_cache - - doc = tmp_path / "doc.md" - doc.write_text("# Doc\n") - spec = tmp_path / "extraction-spec.md" - spec.write_text("PROMPT V1", encoding="utf-8") - save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], - root=tmp_path, prompt_file=str(spec)) - files_from = tmp_path / "files.txt" - files_from.write_text(str(doc) + "\n") - monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) - - base = ["graphify", "cache-check", str(files_from), "--root", str(tmp_path)] - _run_extract(monkeypatch, base + ["--prompt-file", str(spec)]) - assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out - - # An upgrade rewrites the prompt: the entry must no longer satisfy the run. - spec.write_text("PROMPT V2 — rewritten by an upgrade", encoding="utf-8") - os.utime(spec, ns=(0, 0)) - _run_extract(monkeypatch, base + ["--prompt-file", str(spec)]) - assert "Cache: 0 hit, 1 miss" in capsys.readouterr().out diff --git a/tests/test_extraction_spec_ids.py b/tests/test_extraction_spec_ids.py index 46fabc3ba..8a931262f 100644 --- a/tests/test_extraction_spec_ids.py +++ b/tests/test_extraction_spec_ids.py @@ -38,7 +38,8 @@ def _spec_files() -> list[Path]: for p in root.rglob("extraction-spec.md"): # build/ is a packaging artifact; expected/ is skillgen's own golden # output and is already covered by `skillgen --check`. - if "/build/" in p.as_posix() or "/expected/" in p.as_posix(): + relative = p.relative_to(REPO_ROOT).parts + if "build" in relative or "expected" in relative: continue files.append(p) return sorted(files) diff --git a/tests/test_security.py b/tests/test_security.py index 74669f938..113f4dd11 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -27,14 +27,18 @@ _sanitize_metadata_value, ) +from .test_utils import skip_in_sandbox + # --------------------------------------------------------------------------- # validate_url # --------------------------------------------------------------------------- +@skip_in_sandbox() def test_validate_url_accepts_http(): assert validate_url("http://example.com/page") == "http://example.com/page" +@skip_in_sandbox() def test_validate_url_accepts_https(): assert validate_url("https://arxiv.org/abs/1706.03762") == "https://arxiv.org/abs/1706.03762" @@ -78,6 +82,7 @@ def test_safe_fetch_rejects_ftp_url(): with pytest.raises(ValueError, match="ftp"): safe_fetch("ftp://example.com/file.zip") +@skip_in_sandbox() def test_safe_fetch_returns_bytes(tmp_path): mock_resp = _make_mock_response(b"hello world") with patch("graphify.security._build_opener") as mock_opener_fn: @@ -87,6 +92,7 @@ def test_safe_fetch_returns_bytes(tmp_path): result = safe_fetch("https://example.com/") assert result == b"hello world" +@skip_in_sandbox() def test_safe_fetch_raises_on_non_2xx(): mock_resp = _make_mock_response(b"Not Found", status=404) with patch("graphify.security._build_opener") as mock_opener_fn: @@ -96,6 +102,7 @@ def test_safe_fetch_raises_on_non_2xx(): with pytest.raises(urllib.error.HTTPError): safe_fetch("https://example.com/missing") +@skip_in_sandbox() def test_safe_fetch_raises_on_size_exceeded(): # Build a response larger than max_bytes big_chunk = b"x" * 65_537 @@ -119,6 +126,7 @@ def test_safe_fetch_raises_on_size_exceeded(): # safe_fetch_text # --------------------------------------------------------------------------- +@skip_in_sandbox() def test_safe_fetch_text_decodes_utf8(): content = "héllo wörld".encode("utf-8") mock_resp = _make_mock_response(content) @@ -129,6 +137,7 @@ def test_safe_fetch_text_decodes_utf8(): result = safe_fetch_text("https://example.com/") assert result == "héllo wörld" +@skip_in_sandbox() def test_safe_fetch_text_replaces_bad_bytes(): bad = b"hello \xff world" mock_resp = _make_mock_response(bad) diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 0c09e601e..483d0f6ff 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -13,6 +13,8 @@ import pytest +from .test_utils import skip_in_sandbox + # tests/ -> repo root is one parent up; put it on the path so tools.skillgen # imports regardless of pytest's import mode. REPO_ROOT = Path(__file__).resolve().parent.parent @@ -22,6 +24,7 @@ from tools.skillgen import gen # noqa: E402 +@skip_in_sandbox() def test_audit_coverage_passes(): """Every v8 heading lands in the lean core or exactly one reference.""" platforms = gen.load_platforms() @@ -248,6 +251,7 @@ def test_check_passes_for_codex_and_windows(): assert problems == [], f"[{key}]\n" + "\n".join(problems) +@skip_in_sandbox() def test_audit_coverage_passes_for_codex_and_windows(): """Every v8 heading single-homes for the cli-inline split hosts too.""" platforms = gen.load_platforms() @@ -388,6 +392,7 @@ def test_schema_singleton_catches_legacy_enums(): ) +@skip_in_sandbox() def test_all_progressive_hosts_check_and_audit_clean(): """check + audit-coverage pass for every rendered progressive host.""" platforms = gen.load_platforms() @@ -473,6 +478,7 @@ def test_monoliths_render_inline_single_file_no_references(): assert "references/" not in arts[0].content or "see `references/" not in arts[0].content.lower() +@skip_in_sandbox() def test_monolith_roundtrip_passes_for_aider_and_devin(): """Each monolith is diff-clean vs v8 except the file_type enum unification.""" platforms = gen.load_platforms() @@ -481,6 +487,7 @@ def test_monolith_roundtrip_passes_for_aider_and_devin(): assert problems == [], f"[{key}]\n" + "\n".join(problems) +@skip_in_sandbox() def test_monoliths_change_only_sanctioned_lines(): """Every line that differs from pristine v8 is a sanctioned change-class. @@ -607,6 +614,7 @@ def test_always_on_included_in_full_render_not_per_platform(): assert "graphify/always_on/claude-md.md" not in claude_only +@skip_in_sandbox() def test_always_on_roundtrip_is_byte_faithful(): """Each always_on/*.md reproduces its former __main__.py constant byte for byte. @@ -685,6 +693,7 @@ def test_always_on_files_are_guarded_by_check(tmp_path): # --- the per-host coverage audit (the systemic guard) -------------------------- +@skip_in_sandbox() def test_audit_coverage_passes_for_every_split_host(): """Every split host's render single-homes its own v8 body's headings.""" platforms = gen.load_platforms() @@ -705,6 +714,7 @@ def test_audit_reads_each_host_against_its_own_v8_body(): assert gen._v8_baseline_ref("vscode") == "47042beb05d1f6dd2186c0c499ae2840ce604ead:graphify/skill-vscode.md" +@skip_in_sandbox() def test_audit_catches_an_induced_per_host_drop(): """Re-inducing the trae regression (claude-flavored hooks) fails the audit. @@ -722,6 +732,7 @@ def test_audit_catches_an_induced_per_host_drop(): assert any("native AGENTS.md integration (Trae)" in p for p in problems), problems +@skip_in_sandbox() def test_audit_catches_a_dropped_non_allowlisted_heading(): """A core fragment that drops a real v8 heading fails the audit. @@ -882,6 +893,7 @@ def test_amp_has_no_pretooluse_caveat_anywhere(): assert "Trae" not in b2 +@skip_in_sandbox() def test_amp_audit_coverage_passes_against_its_own_v8(): """The per-host audit (the guard amp is the exact case for) passes for amp. @@ -943,6 +955,7 @@ def test_agents_body_matches_amp_modulo_hooks_wording(): assert amp["hooks.md"] != agents["hooks.md"] +@skip_in_sandbox() def test_agents_audit_baseline_is_amps_v8_body(): """`agents` is a post-v8 platform, so its audit baseline is amp's v8 body.""" platforms = gen.load_platforms() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..26f21ce6f --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,14 @@ +"""Shared test utilities and markers.""" + +import os + +import pytest + + +def skip_in_sandbox(): + """Skip tests that need access to external resources (git, network) in a sandbox.""" + sandbox_variables = ["NIX_ENFORCE_PURITY"] + return pytest.mark.skipif( + any(var in os.environ for var in sandbox_variables), + reason="Sandboxed environment with limited external access" + ) diff --git a/uv.lock b/uv.lock index 088ebbbdc..2fd137000 100644 --- a/uv.lock +++ b/uv.lock @@ -1090,7 +1090,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.9.6" +version = "0.9.13" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -1141,6 +1141,7 @@ all = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "openai" }, { name = "openpyxl" }, + { name = "psycopg", extra = ["binary"] }, { name = "pypdf" }, { name = "python-docx" }, { name = "starlette" }, @@ -1280,6 +1281,7 @@ requires-dist = [ { name = "openpyxl", marker = "extra == 'all'" }, { name = "openpyxl", marker = "extra == 'google'" }, { name = "openpyxl", marker = "extra == 'office'" }, + { name = "psycopg", extras = ["binary"], marker = "extra == 'all'" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'" }, { name = "pypdf", marker = "extra == 'all'", specifier = ">=6.12.0" }, { name = "pypdf", marker = "extra == 'pdf'", specifier = ">=6.12.0" },