diff --git a/README.md b/README.md index 81685cf72..879a160bc 100644 --- a/README.md +++ b/README.md @@ -507,6 +507,7 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `AWS_*` / `~/.aws/credentials` | AWS Bedrock — standard credential chain | `--backend bedrock` (no API key, uses IAM) | | `GRAPHIFY_MAX_WORKERS` | AST parallelism thread count | optional — also `--max-workers` flag | | `GRAPHIFY_MAX_OUTPUT_TOKENS` | Raise output cap for dense corpora | optional — e.g. `32768` for large files | +| `GRAPHIFY_FILE_CHAR_CAP` | Per-file input cap and slice size for oversized documents | optional — default `20000`; e.g. `60000` for fewer, larger slices | | `GRAPHIFY_API_TIMEOUT` | Per-call timeout in seconds for HTTP, claude-cli, and Anthropic SDK backends (default: 600) | optional — also `--api-timeout` flag | | `GRAPHIFY_MAX_RETRIES` | How many times to retry a rate-limited (429) request before giving up (default: 6; honors `Retry-After`) | optional — raise for strict per-org limits (e.g. kimi); `0` disables | | `GRAPHIFY_FORCE` | Force graph rebuild even with fewer nodes | optional — also `--force` flag | @@ -718,6 +719,7 @@ graphify extract ./docs --force # overwrite graph.json even if ne graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) graphify extract ./docs --global --as myrepo # extract and register into the cross-project global graph GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # raise output cap for dense corpora +GRAPHIFY_FILE_CHAR_CAP=60000 graphify extract ./docs --backend claude # 60k chars per file/slice (fewer, larger slices) graphify export callflow-html # graphify-out/-callflow.html graphify export callflow-html --max-sections 8 # cap generated architecture sections diff --git a/graphify/llm.py b/graphify/llm.py index e3dbf681b..7f95fbdbc 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -273,6 +273,19 @@ def _resolve_max_tokens(default: int) -> int: return default +def _resolve_file_char_cap(default: int = _FILE_CHAR_CAP) -> int: + """Honour GRAPHIFY_FILE_CHAR_CAP env var override, else use default.""" + raw = os.environ.get("GRAPHIFY_FILE_CHAR_CAP", "").strip() + if raw: + try: + value = int(raw) + if value > 0: + return value + except ValueError: + pass + return default + + # Model-name fragments for OpenAI-compatible "reasoning" models that reject an # explicit temperature: the API returns 400 "Unsupported value: 'temperature' # does not support 0 with this model. Only the default (1) value is supported." @@ -532,6 +545,7 @@ def _read_files(units: "list[Path | FileSlice]", root: Path) -> str: source_file and the graph isn't fragmented per-slice. """ parts: list[str] = [] + file_char_cap = _resolve_file_char_cap() for u in units: p = unit_path(u) safe_path = _resolve_under_root(p, root) @@ -551,7 +565,7 @@ def _read_files(units: "list[Path | FileSlice]", root: Path) -> str: continue # Whole files are still capped (covers non-splittable large files like # code); slices are already bounded to the cap, so the cap is a no-op. - parts.append(_wrap_untrusted(rel, content[:_FILE_CHAR_CAP])) + parts.append(_wrap_untrusted(rel, content[:file_char_cap])) return "\n\n".join(parts) @@ -1526,17 +1540,19 @@ def _estimate_file_tokens(unit: "Path | FileSlice") -> int: """Estimate the prompt-token cost of a file or slice under `_read_files` rules. Uses tiktoken (`cl100k_base`) when available for accurate counts. Falls back - to the chars/4 heuristic if tiktoken is not installed. Both paths cap at - `_FILE_CHAR_CAP` to match `_read_files`'s truncation, plus a constant for - the wrapper. Returns 0 for unreadable paths so they don't blow up packing. + to the chars/4 heuristic if tiktoken is not installed. Both paths use the + resolved file-character cap to match `_read_files`'s truncation, plus a + constant for the wrapper. Returns 0 for unreadable paths so they don't + blow up packing. """ + file_char_cap = _resolve_file_char_cap() if isinstance(unit, FileSlice): - # A slice's size is its char range (already ≤ _FILE_CHAR_CAP). Use the + # A slice's size is its char range (already ≤ the resolved cap). Use the # tokenizer on its text when available, else the chars/4 heuristic. if _TOKENIZER is None: - return (min(unit.end - unit.start, _FILE_CHAR_CAP) + _PER_FILE_OVERHEAD_CHARS) // _CHARS_PER_TOKEN + return (min(unit.end - unit.start, file_char_cap) + _PER_FILE_OVERHEAD_CHARS) // _CHARS_PER_TOKEN try: - content = read_slice_text(unit)[:_FILE_CHAR_CAP] + content = read_slice_text(unit)[:file_char_cap] except OSError: return 0 return len(_TOKENIZER.encode(content, disallowed_special=())) + (_PER_FILE_OVERHEAD_CHARS // _CHARS_PER_TOKEN) @@ -1551,11 +1567,11 @@ def _estimate_file_tokens(unit: "Path | FileSlice") -> int: size = path.stat().st_size except OSError: return 0 - chars = min(size, _FILE_CHAR_CAP) + _PER_FILE_OVERHEAD_CHARS + chars = min(size, file_char_cap) + _PER_FILE_OVERHEAD_CHARS return chars // _CHARS_PER_TOKEN try: - content = path.read_text(encoding="utf-8", errors="replace")[:_FILE_CHAR_CAP] + content = path.read_text(encoding="utf-8", errors="replace")[:file_char_cap] except OSError: return 0 return len(_TOKENIZER.encode(content, disallowed_special=())) + (_PER_FILE_OVERHEAD_CHARS // _CHARS_PER_TOKEN) @@ -1862,9 +1878,12 @@ def extract_corpus_parallel( """ files = [f if isinstance(f, (Path, FileSlice)) else Path(f) for f in files] # Split oversized splittable documents into slices that cover the whole file - # before packing, so content past _FILE_CHAR_CAP is extracted instead of + # before packing, so content past the per-file cap is extracted instead of # silently dropped (#1369). Files at/under the cap pass through unchanged. - files = expand_oversized_files(files, _FILE_CHAR_CAP) + # The cap doubles as the slice size, so GRAPHIFY_FILE_CHAR_CAP trades + # fewer, larger slices (more intra-document context per call) for bigger + # per-call prompts. + files = expand_oversized_files(files, _resolve_file_char_cap()) if token_budget is not None: chunks = _pack_chunks_by_tokens(files, token_budget=token_budget) else: diff --git a/tests/test_file_char_cap.py b/tests/test_file_char_cap.py new file mode 100644 index 000000000..b70bdcd0a --- /dev/null +++ b/tests/test_file_char_cap.py @@ -0,0 +1,61 @@ +"""Regression tests for the per-file semantic extraction character cap.""" + +from graphify import llm +from graphify.file_slice import FileSlice, expand_oversized_files + + +def test_file_char_cap_default_stays_at_twenty_thousand(monkeypatch, tmp_path): + source = tmp_path / "long.md" + source.write_text("x" * 25_000, encoding="utf-8") + monkeypatch.delenv("GRAPHIFY_FILE_CHAR_CAP", raising=False) + + prompt = llm._read_files([source], tmp_path) + + assert prompt.count("x") == 20_000 + + +def test_file_char_cap_env_controls_reading_and_token_estimate(monkeypatch, tmp_path): + source = tmp_path / "long.md" + source.write_text("x" * 25_000, encoding="utf-8") + monkeypatch.setenv("GRAPHIFY_FILE_CHAR_CAP", "30000") + monkeypatch.setattr(llm, "_TOKENIZER", None) + + prompt = llm._read_files([source], tmp_path) + + assert prompt.count("x") == 25_000 + assert llm._estimate_file_tokens(source) == ( + 25_000 + llm._PER_FILE_OVERHEAD_CHARS + ) // llm._CHARS_PER_TOKEN + + +def test_resolve_file_char_cap_falls_back_for_invalid_values(monkeypatch): + for value in ("", "invalid", "0", "-1"): + monkeypatch.setenv("GRAPHIFY_FILE_CHAR_CAP", value) + assert llm._resolve_file_char_cap(20_000) == 20_000 + + +def test_env_cap_doubles_as_slice_size(monkeypatch, tmp_path): + """The resolved cap is the slice size: raising it merges slices back.""" + source = tmp_path / "long.md" + source.write_text("x" * 25_000, encoding="utf-8") + + monkeypatch.delenv("GRAPHIFY_FILE_CHAR_CAP", raising=False) + units = expand_oversized_files([source], llm._resolve_file_char_cap()) + assert all(isinstance(u, FileSlice) for u in units) + assert len(units) == 2 + + monkeypatch.setenv("GRAPHIFY_FILE_CHAR_CAP", "30000") + units = expand_oversized_files([source], llm._resolve_file_char_cap()) + assert units == [source] + + +def test_estimate_slice_tokens_uses_resolved_cap(monkeypatch, tmp_path): + source = tmp_path / "long.md" + source.write_text("x" * 25_000, encoding="utf-8") + monkeypatch.setenv("GRAPHIFY_FILE_CHAR_CAP", "30000") + monkeypatch.setattr(llm, "_TOKENIZER", None) + + fs = FileSlice(path=source, start=0, end=25_000, index=0, total=1) + assert llm._estimate_file_tokens(fs) == ( + 25_000 + llm._PER_FILE_OVERHEAD_CHARS + ) // llm._CHARS_PER_TOKEN diff --git a/uv.lock b/uv.lock index 088ebbbdc..09653a59d 100644 --- a/uv.lock +++ b/uv.lock @@ -1090,7 +1090,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.9.6" +version = "0.9.17" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },