From d0c0917fe4a9226eecce0e2b948aa969d5c74035 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 18:38:01 +0200 Subject: [PATCH 01/18] feat!: flatten the indexing config and honour --config everywhere An explicit-path load_config() now becomes the process-wide active configuration that cfgload.get() answers from, so `--config X` no longer silently takes timeouts, batching, parallelism and the .doc scratch location from an auto-discovered file. The CLI activates the given path before importing the index and search modules, and every duplicated per-call-site fallback default (including the 8-vs-4 parallel_workers disagreement) goes away. Schema changes: - indexing.timeout.* flattened to indexing.per_file_seconds (0 disables) and indexing.scan_item_seconds; the heartbeat and .doc conversion budgets are internal now - indexing.enable_adaptive_batching / max_memory_mb folded into batch_size (an upper bound over the adaptive sizer); enable_parallel_loading folded into parallel_workers (1 = serial) - indexing.logging.slow_file_threshold_seconds flattened to indexing.slow_file_threshold_seconds; log_slow_files was never read - retrieval.bm25_similarity_top_k removed (the BM25 top-k was hardcoded) - include/exclude pattern lists defined once, in cfgload - an empty YAML value (chunk_size:) keeps the default instead of loading None, and the returned config never aliases the defaults - a config file that still sets a removed option gets a warning naming its replacement BREAKING CHANGE: the indexing.timeout, indexing.logging, enable_parallel_loading, enable_adaptive_batching, max_memory_mb and retrieval.bm25_similarity_top_k options are gone; the loader warns about each one it finds and names the replacement. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- README.md | 19 ++--- config.yaml | 31 +++----- src/chunksilo/cfgload.py | 149 ++++++++++++++++++++++-------------- src/chunksilo/cli.py | 8 +- src/chunksilo/docx_utils.py | 2 +- src/chunksilo/index.py | 81 ++++++++------------ src/chunksilo/ui.py | 4 +- test/conftest.py | 15 ++++ test/test_cfgload.py | 129 +++++++++++++++++++++++++++++++ 9 files changed, 293 insertions(+), 145 deletions(-) create mode 100644 test/test_cfgload.py diff --git a/README.md b/README.md index 2d32476..4778215 100644 --- a/README.md +++ b/README.md @@ -151,22 +151,17 @@ All settings are optional and have sensible defaults. | `indexing.defaults.exclude` | Default exclude patterns for all directories | | `indexing.defaults.recursive` | Default recursive setting for all directories | -**Advanced indexing options** — performance tuning, timeouts, and logging: +**Advanced indexing options** — performance tuning, timeouts, and diagnostics: | Setting | Description | | :--- | :--- | -| `indexing.parallel_workers` | Number of threads for parallel file loading | -| `indexing.enable_parallel_loading` | Enable/disable parallel file loading | -| `indexing.enable_adaptive_batching` | Enable memory-aware adaptive batch sizing | -| `indexing.max_memory_mb` | Memory budget (MB) for adaptive batch sizing | +| `indexing.parallel_workers` | Threads for parallel file loading (1 = serial) | +| `indexing.batch_size` | Upper bound on the embedding batch size; sizing adapts down under memory pressure | | `indexing.checkpoint_interval_files` | Files processed between index checkpoints | | `indexing.checkpoint_interval_seconds` | Seconds between index checkpoints | -| `indexing.timeout.enabled` | Enable per-file processing timeout | -| `indexing.timeout.per_file_seconds` | Timeout in seconds for processing each file | -| `indexing.timeout.doc_conversion_seconds` | Timeout in seconds for .doc to .docx conversion | -| `indexing.timeout.heartbeat_interval_seconds` | Interval (seconds) between progress animation updates during file processing | -| `indexing.logging.log_slow_files` | Warn when files take unusually long to process | -| `indexing.logging.slow_file_threshold_seconds` | Seconds before a file is considered slow | +| `indexing.per_file_seconds` | Give up on a single file after this many seconds (0 disables) | +| `indexing.scan_item_seconds` | Timeout for stat/hash/walk operations while scanning | +| `indexing.slow_file_threshold_seconds` | Warn when one file takes longer than this | #### Retrieval Settings @@ -180,7 +175,7 @@ All settings are optional and have sensible defaults. | `retrieval.score_threshold` | Minimum score (0.0-1.0) for results | | `retrieval.recency_boost` | Recency boost weight (0.0-1.0) | | `retrieval.recency_half_life_days` | Days until recency boost halves | -| `retrieval.bm25_similarity_top_k` | Files returned by BM25 filename search | + | `retrieval.offline` | Prevent ML library network requests | #### Confluence Settings (optional) diff --git a/config.yaml b/config.yaml index 744d813..cf42b26 100644 --- a/config.yaml +++ b/config.yaml @@ -1,14 +1,14 @@ # ChunkSilo configuration -# See all options and defaults: chunksilo --dump-defaults +# All options and their defaults: chunksilo --dump-defaults -# Indexing settings - used by chunksilo --build-index +# What to index - used by: chunksilo --build-index indexing: directories: - "./data" # chunk_size: 512 # chunk_overlap: 50 - ## Per-directory defaults (applied to all directories unless overridden) + ## Per-directory defaults (applied to every directory unless overridden per entry) # defaults: # include: ["**/*.pdf", "**/*.md", "**/*.txt", "**/*.docx", "**/*.doc"] # exclude: @@ -26,26 +26,18 @@ indexing: # recursive: true # case_sensitive: false - ## Performance tuning - # batch_size: 200 - # parallel_workers: 8 - # enable_parallel_loading: true - # enable_adaptive_batching: true - # max_memory_mb: 2048 + ## Performance + # batch_size: 200 # embedding batch size (adapts down under memory pressure) + # parallel_workers: 8 # 1 = load files serially # checkpoint_interval_files: 500 # checkpoint_interval_seconds: 300 - ## Timeout protection - # timeout: - # enabled: true - # per_file_seconds: 300 - # doc_conversion_seconds: 90 - # heartbeat_interval_seconds: 2 + ## Timeouts, in seconds + # per_file_seconds: 300 # give up on a single file after this long (0 disables) + # scan_item_seconds: 30 # stat/hash/walk operations while scanning - ## Logging - # logging: - # log_slow_files: true - # slow_file_threshold_seconds: 30 + ## Diagnostics + # slow_file_threshold_seconds: 30 # warn when one file takes longer than this # Confluence integration (optional) - enables live search of Confluence pages # Installed by default; set the values below to enable. @@ -95,7 +87,6 @@ retrieval: # score_threshold: 0.1 # recency_boost: 0.3 # recency_half_life_days: 365 - # bm25_similarity_top_k: 10 # Storage paths storage: diff --git a/src/chunksilo/cfgload.py b/src/chunksilo/cfgload.py index ee172b4..2ac44ed 100644 --- a/src/chunksilo/cfgload.py +++ b/src/chunksilo/cfgload.py @@ -3,7 +3,11 @@ """ Shared configuration loading for ChunkSilo. -Loads configuration from config.yaml, searching in standard locations. +Loads configuration from config.yaml, searching in standard locations. The +first load with an explicit path (``--config``) becomes the process-wide +active configuration: every later no-argument ``load_config()`` or ``get()`` +call answers from it, so a path given on the command line governs the whole +process rather than only the call sites it was threaded through. """ import copy import logging @@ -46,47 +50,41 @@ def _find_config() -> Path: CONFIG_PATH = _find_config() +# Default file type patterns - the single definition; index.py imports these. +DEFAULT_INCLUDE_PATTERNS = ["**/*.pdf", "**/*.md", "**/*.txt", "**/*.docx", "**/*.doc"] + +DEFAULT_EXCLUDE_PATTERNS = [ + "**/.git/**", + "**/node_modules/**", + "**/__pycache__/**", + "**/.venv/**", + "**/venv/**", + "**/.tox/**", + "**/.mypy_cache/**", + "**/.pytest_cache/**", + "**/.eggs/**", + "**/*.egg-info/**", + "**/.DS_Store", +] + _DEFAULTS: dict[str, Any] = { "indexing": { "directories": ["./data"], "defaults": { - "include": ["**/*.pdf", "**/*.md", "**/*.txt", "**/*.docx", "**/*.doc"], - "exclude": [ - "**/.git/**", - "**/node_modules/**", - "**/__pycache__/**", - "**/.venv/**", - "**/venv/**", - "**/.tox/**", - "**/.mypy_cache/**", - "**/.pytest_cache/**", - "**/.eggs/**", - "**/*.egg-info/**", - "**/.DS_Store", - ], + "include": DEFAULT_INCLUDE_PATTERNS, + "exclude": DEFAULT_EXCLUDE_PATTERNS, "recursive": True, "case_sensitive": False, }, "chunk_size": 512, "chunk_overlap": 50, - "batch_size": 200, + "batch_size": 200, # upper bound; adaptive sizing may shrink a batch + "parallel_workers": 8, # 1 = load files serially "checkpoint_interval_files": 500, "checkpoint_interval_seconds": 300, - "parallel_workers": 8, - "enable_parallel_loading": True, - "enable_adaptive_batching": True, - "max_memory_mb": 2048, - "timeout": { - "enabled": True, - "per_file_seconds": 300, # 5 minutes per file - "doc_conversion_seconds": 90, # 90 seconds for .doc conversion - "heartbeat_interval_seconds": 2, - "scan_item_seconds": 30, # timeout for stat/hash/walk during scanning - }, - "logging": { - "log_slow_files": True, - "slow_file_threshold_seconds": 30, - }, + "per_file_seconds": 300, # give up on a single file; 0 disables + "scan_item_seconds": 30, # timeout for stat/hash/walk during scanning + "slow_file_threshold_seconds": 30, # warn when one file takes longer }, "retrieval": { "embed_model_name": "BAAI/bge-small-en-v1.5", @@ -97,7 +95,6 @@ def _find_config() -> Path: "score_threshold": 0.1, "recency_boost": 0.3, "recency_half_life_days": 365, - "bm25_similarity_top_k": 10, "offline": False, }, "confluence": { @@ -118,6 +115,8 @@ def _find_config() -> Path: "include_custom_fields": False, }, "ssl": { + # CA bundle for environments with TLS interception or a private CA: + # used for model downloads and for the Confluence/Jira APIs. "ca_bundle_path": "", }, "storage": { @@ -126,15 +125,44 @@ def _find_config() -> Path: }, } +# Options that no longer exist, mapped to what replaced them. A config file +# that still sets one gets a warning instead of silently falling back to the +# built-in default. +_REMOVED_KEYS = { + "indexing.timeout": "indexing.per_file_seconds (0 disables) and indexing.scan_item_seconds", + "indexing.logging": "indexing.slow_file_threshold_seconds", + "indexing.enable_parallel_loading": "indexing.parallel_workers (1 = serial)", + "indexing.enable_adaptive_batching": "indexing.batch_size (an upper bound; sizing adapts automatically)", + "indexing.max_memory_mb": "indexing.batch_size (an upper bound; sizing adapts automatically)", + "retrieval.bm25_similarity_top_k": "nothing; the number of file-name matches is fixed", +} + +# The active configuration: the merged result of the last load, plus the path +# it came from. An explicit-path load replaces it; no-argument loads reuse it. _config_cache: dict[str, Any] | None = None +_active_path: Path | None = None + + +def _warn_removed_keys(user_config: dict[str, Any], path: Path) -> None: + """Warn about options the file sets that the schema no longer has.""" + for dotted, replacement in _REMOVED_KEYS.items(): + section, _, key = dotted.partition(".") + block = user_config.get(section) + if isinstance(block, dict) and key in block: + logger.warning( + "%s: '%s' is no longer used and is ignored; use %s instead", + path, dotted, replacement, + ) def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: """Deep merge override into base, returning a new dict.""" result = base.copy() for key, value in override.items(): - if value is None and key in result and isinstance(result[key], dict): - continue # Skip None overrides for dict sections (e.g., commented-out YAML) + if value is None and key in result: + # An empty YAML value ("chunk_size:" or a commented-out section) + # parses as None; keep the default instead of poisoning it. + continue if key in result and isinstance(result[key], dict) and isinstance(value, dict): result[key] = _deep_merge(result[key], value) else: @@ -182,45 +210,49 @@ def _resolve_bundled_models(config: dict[str, Any]) -> dict[str, Any]: return result -def load_config(config_path: Path | None = None) -> dict[str, Any]: +def load_config(config_path: Path | str | None = None) -> dict[str, Any]: """Load configuration from YAML file with defaults. Args: - config_path: Optional path to config file. If None, uses default CONFIG_PATH. - Results are cached only when using the default path. + config_path: Optional path to config file. When given, the result + becomes the process-wide active configuration that + no-argument calls (and :func:`get`) answer from. When + None, returns the active configuration, loading from the + auto-discovered CONFIG_PATH on first use. Returns: Configuration dictionary with defaults merged in. """ - global _config_cache + global _config_cache, _active_path - # Return cached config if available (only for default path) - if _config_cache is not None and config_path is None: + if _config_cache is not None and ( + config_path is None or Path(config_path) == _active_path + ): return _config_cache - path = config_path or CONFIG_PATH + path = Path(config_path) if config_path else CONFIG_PATH + + # Callers may mutate the returned dict; never hand out the defaults themselves. + defaults = copy.deepcopy(_DEFAULTS) if not path.exists(): logger.info("Config file not found at %s; using built-in defaults", path) - return _resolve_bundled_models(_DEFAULTS.copy()) - - logger.info("Using config: %s", path) - - with open(path, encoding="utf-8") as f: - user_config = yaml.safe_load(f) or {} - - result = _deep_merge(_DEFAULTS, user_config) - result = _resolve_bundled_models(result) - - # Cache result only for default path - if config_path is None: - _config_cache = result - + result = _resolve_bundled_models(defaults) + else: + logger.info("Using config: %s", path) + with open(path, encoding="utf-8") as f: + user_config = yaml.safe_load(f) or {} + _warn_removed_keys(user_config, path) + result = _deep_merge(defaults, user_config) + result = _resolve_bundled_models(result) + + _config_cache = result + _active_path = path return result def get(key: str, default: Any = None) -> Any: - """Get a config value by dot-notation key. + """Get a value from the active configuration by dot-notation key. Args: key: Dot-separated key path (e.g., 'retrieval.embed_top_k') @@ -247,7 +279,8 @@ def get(key: str, default: Any = None) -> Any: def reload_config() -> dict[str, Any]: - """Force reload configuration from disk, clearing the cache.""" - global _config_cache + """Force reload configuration from disk, clearing the active config.""" + global _config_cache, _active_path _config_cache = None + _active_path = None return load_config() diff --git a/src/chunksilo/cli.py b/src/chunksilo/cli.py index 8b17bcf..acdfd63 100644 --- a/src/chunksilo/cli.py +++ b/src/chunksilo/cli.py @@ -65,11 +65,15 @@ def main(): config_path = Path(args.config) if args.config else None + # Activate --config for the whole process before importing .index or + # .search, which read module-level settings from the active configuration. + from .cfgload import load_config + + cfg = load_config(config_path) + if args.list_files: - from .cfgload import load_config from .index import IngestionState - cfg = load_config(config_path) state_db = Path(cfg["storage"]["storage_dir"]) / "ingestion_state.db" if not state_db.exists(): print("No index found. Run chunksilo --build-index first.", file=sys.stderr) diff --git a/src/chunksilo/docx_utils.py b/src/chunksilo/docx_utils.py index b7bc5c1..f263163 100644 --- a/src/chunksilo/docx_utils.py +++ b/src/chunksilo/docx_utils.py @@ -48,7 +48,7 @@ def _is_heading_style(style_name: str) -> bool: def _get_doc_temp_dir() -> Path: """Get the temporary directory for .doc conversion, creating it if needed.""" - storage_dir = Path(cfgload.get("storage.storage_dir", "./storage")) + storage_dir = Path(cfgload.get("storage.storage_dir")) temp_dir = storage_dir / "doc_temp" temp_dir.mkdir(parents=True, exist_ok=True) return temp_dir diff --git a/src/chunksilo/index.py b/src/chunksilo/index.py index 17d1db7..8a4457d 100644 --- a/src/chunksilo/index.py +++ b/src/chunksilo/index.py @@ -37,7 +37,7 @@ # Load configuration from config.yaml from . import cfgload -from .cfgload import load_config +from .cfgload import DEFAULT_EXCLUDE_PATTERNS, DEFAULT_INCLUDE_PATTERNS, load_config from .docx_utils import _convert_doc_to_docx, split_docx_into_heading_documents from .models import _get_cached_model_path, configure_offline_mode, resolve_flashrank_model_name from .ui import FileProcessingContext, FileProcessingTimeoutError, GracefulAbort, IndexingUI @@ -89,23 +89,12 @@ logger = logging.getLogger(__name__) -# Default file type patterns -DEFAULT_INCLUDE_PATTERNS = ["**/*.pdf", "**/*.md", "**/*.txt", "**/*.docx", "**/*.doc"] - -# Default directory/file exclusion patterns — skip common noise directories -DEFAULT_EXCLUDE_PATTERNS = [ - "**/.git/**", - "**/node_modules/**", - "**/__pycache__/**", - "**/.venv/**", - "**/venv/**", - "**/.tox/**", - "**/.mypy_cache/**", - "**/.pytest_cache/**", - "**/.eggs/**", - "**/*.egg-info/**", - "**/.DS_Store", -] +# Internal tuning that used to be configuration. The adaptive batch sizer works +# against this memory budget, the heartbeat drives the per-file progress +# animation, and .doc conversion gets its own LibreOffice budget. +_MAX_MEMORY_MB = 2048 +_HEARTBEAT_INTERVAL_SECONDS = 2.0 +_DOC_CONVERSION_TIMEOUT_SECONDS = 90.0 @dataclass @@ -614,7 +603,7 @@ def is_available(self) -> bool: Runs with a timeout to avoid hanging on unresponsive network mounts. """ - timeout = cfgload.get("indexing.timeout.scan_item_seconds", 30) + timeout = cfgload.get("indexing.scan_item_seconds") def _check(): try: @@ -704,7 +693,7 @@ def _walk_with_timeout(self): If no result arrives within scan_item_seconds, the walk is considered stalled and iteration stops. """ - timeout = cfgload.get("indexing.timeout.scan_item_seconds", 30) + timeout = cfgload.get("indexing.scan_item_seconds") q: queue.Queue = queue.Queue() _sentinel = None # signals end of iteration @@ -787,7 +776,7 @@ def _create_file_info( Raises TimeoutError if the operation exceeds scan_item_seconds. """ - timeout = cfgload.get("indexing.timeout.scan_item_seconds", 30) + timeout = cfgload.get("indexing.scan_item_seconds") result = _run_with_timeout( lambda: self._create_file_info_inner(file_path, tracked_files), timeout_seconds=timeout, @@ -844,7 +833,7 @@ def load_file( ctx: "FileProcessingContext | None" = None ) -> list[LlamaIndexDocument]: file_path = Path(file_info.path) - exists_timeout = cfgload.get("indexing.timeout.scan_item_seconds", 30) + exists_timeout = cfgload.get("indexing.scan_item_seconds") exists_result = _run_with_timeout( file_path.exists, timeout_seconds=exists_timeout, default=False, ) @@ -884,9 +873,9 @@ def load_file( if ctx: ctx.set_phase("Converting .doc to .docx") - # Use specialized timeout for .doc conversion - doc_timeout = cfgload.get("indexing.timeout.doc_conversion_seconds", 90) - docx_path = _convert_doc_to_docx(file_path, timeout=doc_timeout) + docx_path = _convert_doc_to_docx( + file_path, timeout=_DOC_CONVERSION_TIMEOUT_SECONDS + ) if docx_path is None: logger.warning(f"Skipping {file_path}: could not convert .doc to .docx") @@ -1549,28 +1538,23 @@ def build_index( # Process New/Modified Files if files_to_process: - # Get configuration - timeout_enabled = cfgload.get("indexing.timeout.enabled", True) - per_file_timeout = cfgload.get("indexing.timeout.per_file_seconds", 300) - heartbeat_interval = cfgload.get("indexing.timeout.heartbeat_interval_seconds", 2) + # Get configuration (0 disables the per-file timeout) + per_file_timeout = cfgload.get("indexing.per_file_seconds") + timeout_enabled = bool(per_file_timeout) + heartbeat_interval = _HEARTBEAT_INTERVAL_SECONDS # Checkpointing configuration (controls how often we persist to disk) - checkpoint_interval_files = cfgload.get("indexing.checkpoint_interval_files", 500) - checkpoint_interval_seconds = cfgload.get("indexing.checkpoint_interval_seconds", 300) - - # Batch sizing (controls how many files are loaded/embedded together) - configured_batch_size = cfgload.get("indexing.batch_size", 200) - enable_adaptive_batching = cfgload.get("indexing.enable_adaptive_batching", True) - max_memory_mb = cfgload.get("indexing.max_memory_mb", 2048) - - if enable_adaptive_batching: - optimal_batch_size = calculate_optimal_batch_size( - num_files=len(files_to_process), - max_memory_mb=max_memory_mb - ) - batch_size = min(configured_batch_size, optimal_batch_size) - else: - batch_size = configured_batch_size + checkpoint_interval_files = cfgload.get("indexing.checkpoint_interval_files") + checkpoint_interval_seconds = cfgload.get("indexing.checkpoint_interval_seconds") + + # Batch sizing: the configured size is an upper bound; the adaptive + # sizer shrinks it to fit the memory budget. + configured_batch_size = cfgload.get("indexing.batch_size") + optimal_batch_size = calculate_optimal_batch_size( + num_files=len(files_to_process), + max_memory_mb=_MAX_MEMORY_MB, + ) + batch_size = min(configured_batch_size, optimal_batch_size) # Delete old versions of modified files ui.step_start("Removing old versions of modified files") @@ -1591,9 +1575,8 @@ def build_index( ui.progress_start(total_files, desc="Processing files", unit="file") - # Get parallel loading configuration - max_workers = cfgload.get("indexing.parallel_workers", 4) - enable_parallel = cfgload.get("indexing.enable_parallel_loading", True) + # Parallel loading configuration (1 = serial) + max_workers = max(1, int(cfgload.get("indexing.parallel_workers"))) batch_num = 0 total_batches = (total_files + batch_size - 1) // batch_size @@ -1609,7 +1592,7 @@ def build_index( doc_to_file_mapping = {} file_doc_ids = {} - workers = max_workers if enable_parallel else 1 + workers = max_workers file_docs = load_files_parallel( batch, data_source, diff --git a/src/chunksilo/ui.py b/src/chunksilo/ui.py index 95ddacd..07924b3 100644 --- a/src/chunksilo/ui.py +++ b/src/chunksilo/ui.py @@ -510,9 +510,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): # Warn about slow files (warnings still pass through) if self._start_time: duration = time.time() - self._start_time - slow_threshold = cfgload.get( - "indexing.logging.slow_file_threshold_seconds", 30 - ) + slow_threshold = cfgload.get("indexing.slow_file_threshold_seconds") if duration > slow_threshold: logger.warning( f"Slow file processing: {self.file_path} took {duration:.1f}s" diff --git a/test/conftest.py b/test/conftest.py index 0fb5f22..ae8db23 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -139,3 +139,18 @@ def patched_search_globals(test_env): search._config = original_config search._index_cache = original_index_cache search._embed_model_initialized = original_initialized + + +@pytest.fixture(autouse=True) +def _reset_active_config(): + """Isolate the process-wide active config between tests. + + An explicit-path load_config() call becomes the active configuration for + the whole process; without this reset, one test's tmp config would answer + every later test's no-argument load. + """ + yield + from chunksilo import cfgload + + cfgload._config_cache = None + cfgload._active_path = None diff --git a/test/test_cfgload.py b/test/test_cfgload.py new file mode 100644 index 0000000..e55510f --- /dev/null +++ b/test/test_cfgload.py @@ -0,0 +1,129 @@ +"""Tests for the configuration loader (cfgload.py). + +An explicit-path load becomes the process-wide active configuration, so a +--config argument governs every later cfgload.get() call in the process. +""" +import logging + +import yaml + +from chunksilo import cfgload + + +def _write(path, data): + path.write_text(yaml.safe_dump(data), encoding="utf-8") + return path + + +class TestActiveConfig: + def test_explicit_path_becomes_active(self, tmp_path): + cfg_path = _write(tmp_path / "a.yaml", {"indexing": {"chunk_size": 999}}) + + loaded = cfgload.load_config(cfg_path) + + assert loaded["indexing"]["chunk_size"] == 999 + assert cfgload.get("indexing.chunk_size") == 999 + assert cfgload.load_config() is loaded + assert cfgload._active_path == cfg_path + + def test_string_path_is_accepted(self, tmp_path): + cfg_path = _write(tmp_path / "a.yaml", {"indexing": {"chunk_size": 321}}) + + cfgload.load_config(str(cfg_path)) + + assert cfgload.get("indexing.chunk_size") == 321 + + def test_another_path_replaces_the_active_config(self, tmp_path): + first = _write(tmp_path / "a.yaml", {"indexing": {"chunk_size": 1}}) + second = _write(tmp_path / "b.yaml", {"indexing": {"chunk_size": 2}}) + + cfgload.load_config(first) + cfgload.load_config(second) + + assert cfgload.get("indexing.chunk_size") == 2 + assert cfgload._active_path == second + + def test_same_path_reuses_the_cache(self, tmp_path): + cfg_path = _write(tmp_path / "a.yaml", {"indexing": {"chunk_size": 7}}) + + first = cfgload.load_config(cfg_path) + second = cfgload.load_config(cfg_path) + + assert first is second + + def test_missing_file_uses_defaults_and_is_still_active(self, tmp_path): + missing = tmp_path / "missing.yaml" + + loaded = cfgload.load_config(missing) + + assert loaded["indexing"]["chunk_size"] == cfgload._DEFAULTS["indexing"]["chunk_size"] + assert cfgload._active_path == missing + assert cfgload.load_config() is loaded + + def test_reload_clears_the_active_config(self, tmp_path): + cfg_path = _write(tmp_path / "a.yaml", {"indexing": {"chunk_size": 5}}) + cfgload.load_config(cfg_path) + + cfgload.reload_config() + + assert cfgload._active_path == cfgload.CONFIG_PATH + + def test_returned_config_never_aliases_the_defaults(self, tmp_path): + loaded = cfgload.load_config(tmp_path / "missing.yaml") + + loaded["storage"]["storage_dir"] = "/somewhere/else" + loaded["indexing"]["defaults"]["include"].append("**/*.rst") + + assert cfgload._DEFAULTS["storage"]["storage_dir"] == "./storage" + assert "**/*.rst" not in cfgload.DEFAULT_INCLUDE_PATTERNS + + +class TestMerging: + def test_empty_value_keeps_the_default(self, tmp_path): + cfg_path = tmp_path / "a.yaml" + cfg_path.write_text("indexing:\n chunk_size:\n chunk_overlap: 9\n", encoding="utf-8") + + loaded = cfgload.load_config(cfg_path) + + assert loaded["indexing"]["chunk_size"] == 512 + assert loaded["indexing"]["chunk_overlap"] == 9 + + def test_commented_out_section_keeps_the_defaults(self, tmp_path): + cfg_path = tmp_path / "a.yaml" + cfg_path.write_text("retrieval:\nstorage:\n storage_dir: ./s\n", encoding="utf-8") + + loaded = cfgload.load_config(cfg_path) + + assert loaded["retrieval"]["embed_top_k"] == 20 + assert loaded["storage"]["storage_dir"] == "./s" + + def test_removed_options_are_reported(self, tmp_path, caplog): + cfg_path = _write( + tmp_path / "old.yaml", + { + "indexing": { + "timeout": {"per_file_seconds": 10}, + "enable_adaptive_batching": False, + }, + "retrieval": {"bm25_similarity_top_k": 3}, + }, + ) + + with caplog.at_level(logging.WARNING, logger="chunksilo.cfgload"): + loaded = cfgload.load_config(cfg_path) + + warned = [r.getMessage() for r in caplog.records if "no longer used" in r.getMessage()] + assert len(warned) == 3 + assert any("indexing.timeout" in m and "indexing.per_file_seconds" in m for m in warned) + assert any("indexing.enable_adaptive_batching" in m for m in warned) + assert any("retrieval.bm25_similarity_top_k" in m for m in warned) + # The current schema still answers with its defaults. + assert loaded["indexing"]["per_file_seconds"] == 300 + + def test_current_options_do_not_warn(self, tmp_path, caplog): + cfg_path = _write(tmp_path / "new.yaml", {"indexing": {"per_file_seconds": 10}}) + + with caplog.at_level(logging.WARNING, logger="chunksilo.cfgload"): + cfgload.load_config(cfg_path) + + assert not [r for r in caplog.records if "no longer used" in r.getMessage()] From 44c847982f59fc1850f2b9d6a8b228f43eca9970 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 18:42:35 +0200 Subject: [PATCH 02/18] fix(scripts): stamp the release version into the bundle and download models through the CLI The manylinux bundle installed as the previous version: prepare-common.sh copies pyproject.toml and src/ straight from the checkout, where main still carries the last release's number (the bump lands only after a release finishes), so the tarball name and VERSION file said one thing while pip recorded another. The script now stamps VERSION into the copied sources and verifies it, so a mismatch fails the build instead of shipping. The release scripts and CI also invoked the module-level `python -m chunksilo.index` entry point, a second CLI that duplicated `chunksilo --download-models` with its own --model-cache-dir flag. That entry point and the build_index(model_cache_dir=...) override are gone; the scripts download models through the real CLI with a throwaway config naming the cache location, which the active-config loader honours end to end. setup.sh points at `chunksilo --build-index` as well. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- .github/workflows/manual-release.yml | 6 +++-- .github/workflows/test-functional.yml | 2 +- scripts/prepare-common.sh | 27 +++++++++++++++++-- setup.sh | 2 +- src/chunksilo/index.py | 38 --------------------------- 5 files changed, 31 insertions(+), 44 deletions(-) diff --git a/.github/workflows/manual-release.yml b/.github/workflows/manual-release.yml index f5a10fc..55905ce 100644 --- a/.github/workflows/manual-release.yml +++ b/.github/workflows/manual-release.yml @@ -286,8 +286,10 @@ jobs: # Install the exact dependency set the release used; no upgrades. pip install --no-deps -r "$LOCK_FILE" BUNDLE=src/chunksilo/_bundled_models - PYTHONPATH=src python -m chunksilo.index --download-models \ - --model-cache-dir "$BUNDLE" + DL_CONFIG=$(mktemp) + printf 'storage:\n model_cache_dir: "%s"\n' "$BUNDLE" > "$DL_CONFIG" + PYTHONPATH=src python -m chunksilo --download-models --config "$DL_CONFIG" + rm -f "$DL_CONFIG" # Belt-and-suspenders: materialize any remaining symlinks into real files # (independent of the hf_hub version honoring HF_HUB_DISABLE_SYMLINKS), # then drop the now-redundant blob store. exclude-package-data also keeps diff --git a/.github/workflows/test-functional.yml b/.github/workflows/test-functional.yml index 2ccc09c..0439507 100644 --- a/.github/workflows/test-functional.yml +++ b/.github/workflows/test-functional.yml @@ -62,7 +62,7 @@ jobs: DATA_DIR: ./test_data_dummy STORAGE_DIR: ./test_storage_dummy run: | - python -m chunksilo.index --download-models + python -m chunksilo --download-models - name: Run all functional tests env: diff --git a/scripts/prepare-common.sh b/scripts/prepare-common.sh index 813a4d1..70cef0a 100755 --- a/scripts/prepare-common.sh +++ b/scripts/prepare-common.sh @@ -34,8 +34,31 @@ cp setup.sh "$COMMON_ROOT/" echo "$VERSION" > "$COMMON_ROOT/VERSION" -# Download the embedding + rerank models once (they're the same for all platforms) -PYTHONPATH=src python3.11 -m chunksilo.index --download-models --model-cache-dir "$COMMON_ROOT/models" +# Stamp the release version into the COPIED sources. The checkout still carries +# the previous release's number here, because the version bump on main only +# happens after a release completes - so without this the bundle would install +# as the previous version even though the tarball and VERSION file say otherwise. +# Done on the copies (not the checkout) so the script is safe to run by hand. +sed -i.bak "s/^version = .*/version = \"$VERSION\"/" "$COMMON_ROOT/pyproject.toml" +sed -i.bak "s/^__version__ = .*/__version__ = \"$VERSION\"/" "$COMMON_ROOT/src/chunksilo/__init__.py" +rm -f "$COMMON_ROOT/pyproject.toml.bak" "$COMMON_ROOT/src/chunksilo/__init__.py.bak" + +# Fail loudly rather than ship a bundle whose metadata disagrees with its name. +STAMPED_PROJECT=$(grep -m1 '^version = ' "$COMMON_ROOT/pyproject.toml" | cut -d'"' -f2) +STAMPED_INIT=$(grep -m1 '^__version__ = ' "$COMMON_ROOT/src/chunksilo/__init__.py" | cut -d'"' -f2) +if [ "$STAMPED_PROJECT" != "$VERSION" ] || [ "$STAMPED_INIT" != "$VERSION" ]; then + echo "Error: version stamping failed (pyproject=$STAMPED_PROJECT, __init__=$STAMPED_INIT, expected $VERSION)" >&2 + exit 1 +fi +echo "Stamped version $VERSION into pyproject.toml and __init__.py" + +# Download the embedding + rerank models once (they're the same for all +# platforms). The cache location comes from a throwaway config: the CLI has no +# cache-dir flag, and the config given here becomes the active one. +DL_CONFIG=$(mktemp) +printf 'storage:\n model_cache_dir: "%s"\n' "$COMMON_ROOT/models" > "$DL_CONFIG" +PYTHONPATH=src python3.11 -m chunksilo --download-models --config "$DL_CONFIG" +rm -f "$DL_CONFIG" # Write model license information cat > "$COMMON_ROOT/models/MODEL-LICENSES.txt" << 'MODLICEOF' diff --git a/setup.sh b/setup.sh index 7158867..cbd6acd 100755 --- a/setup.sh +++ b/setup.sh @@ -41,7 +41,7 @@ echo "" echo "Next steps:" echo "1. Edit config.yaml to configure your directories and settings" echo "2. Add documents to ./data (or update indexing.directories in config.yaml)" -echo "3. Build the index: ./venv/bin/python -m chunksilo.index" +echo "3. Build the index: ./venv/bin/chunksilo --build-index" echo "4. Configure your MCP client (see example below)" echo "" echo "=== MCP Client Configuration ===" diff --git a/src/chunksilo/index.py b/src/chunksilo/index.py index 8a4457d..d7f7e3f 100644 --- a/src/chunksilo/index.py +++ b/src/chunksilo/index.py @@ -4,7 +4,6 @@ Indexing pipeline for building a RAG index from PDF, DOCX, DOC, Markdown, and TXT documents. Supports incremental indexing using a local SQLite database to track file states. """ -import argparse import hashlib import itertools import json @@ -1378,7 +1377,6 @@ def load_single_file(file_info: FileInfo): def build_index( download_only: bool = False, config_path: Path | None = None, - model_cache_dir: Path | None = None, verbose: bool = False, ) -> None: """Build and persist the vector index incrementally.""" @@ -1396,9 +1394,6 @@ def build_index( RETRIEVAL_EMBED_MODEL_NAME = cfg["retrieval"]["embed_model_name"] RETRIEVAL_RERANK_MODEL_NAME = cfg["retrieval"]["rerank_model_name"] - # Override model cache dir if specified via CLI - if model_cache_dir: - RETRIEVAL_MODEL_CACHE_DIR = model_cache_dir # Read offline setting from config; force online when downloading models offline = False if download_only else _config["retrieval"].get("offline", False) @@ -1692,36 +1687,3 @@ def build_index( abort_ctl.uninstall() ui.success(f"\nIndexing complete. {len(files_to_process)} files processed.") - -if __name__ == "__main__": - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) - parser = argparse.ArgumentParser(description="Build the document index") - parser.add_argument( - "--download-models", - action="store_true", - help="Download the retrieval models and exit", - ) - parser.add_argument( - "--config", - type=str, - help="Path to config.yaml (overrides auto-discovery)", - ) - parser.add_argument( - "--model-cache-dir", - type=str, - help="Directory to download/cache models (overrides config)", - ) - args = parser.parse_args() - - try: - build_index( - download_only=args.download_models, - config_path=Path(args.config) if args.config else None, - model_cache_dir=Path(args.model_cache_dir) if args.model_cache_dir else None, - ) - except Exception as e: - logger.error(f"Indexing failed: {e}", exc_info=True) - raise From ced6f9bc0b1ba1dab013ec80757c22386d2ad0ed Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 18:48:43 +0200 Subject: [PATCH 03/18] feat: convert .doc files in a private work directory and key their headings by the original path Legacy .doc conversion wrote LibreOffice's output into one shared scratch directory: two parallel workers converting files with the same stem raced for one output path, so one file's content could be indexed under the other file's path. LibreOffice was also handed the original document, which let it drop a .~lock.*# file next to a source inside an indexed directory, and it used the shared profile under $HOME. Each conversion now runs in its own directory under storage on a copy of the source, with a private LibreOffice profile, and the whole directory is removed afterwards. Headings extracted from a converted .doc are stored under the original .doc path, which is where the query side looks them up; previously they were keyed by the temporary .docx path and never found. A new regression test builds an index twice over a populated tree (txt/md/docx/legacy doc, subdirectories with spaces) from a neutral working directory and asserts the tree is byte-for-byte unchanged, with no logs or lock files added. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- src/chunksilo/docx_utils.py | 56 ++++++++++--- src/chunksilo/index.py | 10 ++- test/test_docx_utils.py | 98 ++++++++++++++++++++--- test/test_never_mutates.py | 154 ++++++++++++++++++++++++++++++++++++ 4 files changed, 293 insertions(+), 25 deletions(-) create mode 100644 test/test_never_mutates.py diff --git a/src/chunksilo/docx_utils.py b/src/chunksilo/docx_utils.py index f263163..c5fc1a0 100644 --- a/src/chunksilo/docx_utils.py +++ b/src/chunksilo/docx_utils.py @@ -54,19 +54,29 @@ def _get_doc_temp_dir() -> Path: return temp_dir -def _convert_doc_to_docx(doc_path: Path, timeout: float = 60) -> Path | None: +def _convert_doc_to_docx(doc_path: Path, timeout: float = 90) -> Path | None: """Convert a .doc file to .docx using LibreOffice. + Each conversion gets a private directory under storage: parallel workers + converting two files with the same stem must never share an output path, + or one file's content would be indexed under the other file's path. The + source is copied in first and the copy converted, so LibreOffice never + opens - or drops a ``.~lock.*#`` file next to - anything inside an indexed + directory, and its user profile is pointed into the same private directory + rather than at ``$HOME``. + Args: doc_path: Path to .doc file timeout: Timeout in seconds for conversion process Returns: - Path to temporary .docx file, or None if conversion fails. - Caller is responsible for cleaning up the temp file. + Path to the converted .docx inside a private work directory, or None + if conversion fails. The caller cleans up with + :func:`cleanup_conversion_dir` on the returned path's parent. """ import shutil import subprocess + import tempfile # Find LibreOffice executable soffice_paths = [ @@ -86,23 +96,35 @@ def _convert_doc_to_docx(doc_path: Path, timeout: float = 60) -> Path | None: logger.warning(f"LibreOffice not found. Cannot convert {doc_path}") return None - # Use storage directory for temp files (more reliable space than /tmp) - temp_dir = _get_doc_temp_dir() + # Private per-conversion directory under storage (more reliable space + # than /tmp, and no filename collisions between parallel workers). + work_dir = Path(tempfile.mkdtemp(prefix="conv-", dir=_get_doc_temp_dir())) try: + local_copy = work_dir / doc_path.name + shutil.copyfile(doc_path, local_copy) + result = subprocess.run( - [soffice, "--headless", "--convert-to", "docx", - "--outdir", str(temp_dir), str(doc_path)], + [ + soffice, + "--headless", + # A separate profile per conversion: soffice instances sharing + # one profile serialize on its lock, and a profile under $HOME + # is a write outside the storage directory. + f"-env:UserInstallation={work_dir.joinpath('lo-profile').as_uri()}", + "--convert-to", "docx", + "--outdir", str(work_dir), + str(local_copy), + ], capture_output=True, timeout=timeout, ) if result.returncode != 0: logger.warning(f"LibreOffice conversion failed for {doc_path}: {result.stderr}") + cleanup_conversion_dir(work_dir) return None - # Find the converted file - docx_name = doc_path.stem + ".docx" - docx_path = temp_dir / docx_name + docx_path = work_dir / (doc_path.stem + ".docx") if docx_path.exists(): return docx_path @@ -112,14 +134,23 @@ def _convert_doc_to_docx(doc_path: Path, timeout: float = 60) -> Path | None: except Exception as e: logger.warning(f"Error converting {doc_path}: {e}") + cleanup_conversion_dir(work_dir) return None +def cleanup_conversion_dir(work_dir: Path) -> None: + """Remove a private conversion directory and everything in it.""" + import shutil + + shutil.rmtree(work_dir, ignore_errors=True) + + def split_docx_into_heading_documents( docx_path: Path, ctx: FileProcessingContext | None = None, *, heading_store: Any = None, + heading_store_key: str | None = None, excluded_embed_metadata_keys: list[str] | None = None, excluded_llm_metadata_keys: list[str] | None = None, ) -> list[LlamaIndexDocument]: @@ -129,6 +160,9 @@ def split_docx_into_heading_documents( docx_path: Path to DOCX file ctx: Optional processing context for progress updates and timeout heading_store: HeadingStore instance for persisting heading metadata + heading_store_key: Path to store headings under; defaults to docx_path. + A converted .doc is parsed from a temp copy but looked up at query + time under its original path, so the caller must pass that path. excluded_embed_metadata_keys: Keys to exclude from embedding text excluded_llm_metadata_keys: Keys to exclude from LLM context """ @@ -188,7 +222,7 @@ def split_docx_into_heading_documents( if heading_store is not None: if ctx: ctx.set_phase("Storing heading metadata") - heading_store.set_headings(str(docx_path), all_headings) + heading_store.set_headings(heading_store_key or str(docx_path), all_headings) # Second pass: Split by heading (existing logic) if ctx: diff --git a/src/chunksilo/index.py b/src/chunksilo/index.py index d7f7e3f..a3d5bba 100644 --- a/src/chunksilo/index.py +++ b/src/chunksilo/index.py @@ -37,7 +37,7 @@ # Load configuration from config.yaml from . import cfgload from .cfgload import DEFAULT_EXCLUDE_PATTERNS, DEFAULT_INCLUDE_PATTERNS, load_config -from .docx_utils import _convert_doc_to_docx, split_docx_into_heading_documents +from .docx_utils import _convert_doc_to_docx, cleanup_conversion_dir, split_docx_into_heading_documents from .models import _get_cached_model_path, configure_offline_mode, resolve_flashrank_model_name from .ui import FileProcessingContext, FileProcessingTimeoutError, GracefulAbort, IndexingUI @@ -888,6 +888,7 @@ def load_file( lambda: split_docx_into_heading_documents( docx_path, ctx, heading_store=get_heading_store(), + heading_store_key=str(file_path), excluded_embed_metadata_keys=EXCLUDED_EMBED_METADATA_KEYS, excluded_llm_metadata_keys=EXCLUDED_LLM_METADATA_KEYS, ), @@ -903,6 +904,7 @@ def load_file( docs = split_docx_into_heading_documents( docx_path, ctx, heading_store=get_heading_store(), + heading_store_key=str(file_path), excluded_embed_metadata_keys=EXCLUDED_EMBED_METADATA_KEYS, excluded_llm_metadata_keys=EXCLUDED_LLM_METADATA_KEYS, ) @@ -914,9 +916,9 @@ def load_file( doc.metadata["source"] = str(file_path) return docs finally: - # Clean up temp file - if docx_path.exists(): - docx_path.unlink() + # The whole private conversion directory goes: converted + # .docx, the copied .doc, and the LibreOffice profile. + cleanup_conversion_dir(docx_path.parent) else: reader = SimpleDirectoryReader( input_files=[str(file_path)], diff --git a/test/test_docx_utils.py b/test/test_docx_utils.py index cf3c6c0..b2e7ecb 100644 --- a/test/test_docx_utils.py +++ b/test/test_docx_utils.py @@ -82,6 +82,25 @@ def test_title_style(self): class TestConvertDocToDocx: + """The conversion contract: each conversion runs in a private directory + under storage, on a *copy* of the source, with its own LibreOffice + profile - so parallel same-stem conversions cannot collide (a collision + would index one file's content under another file's path) and nothing is + ever written next to the source document.""" + + @staticmethod + def _fake_soffice(mock_run, produce_output=True): + """Simulate LibreOffice: write .docx into the --outdir.""" + + def run(argv, **kwargs): + if produce_output: + outdir = Path(argv[argv.index("--outdir") + 1]) + source = Path(argv[-1]) + (outdir / (source.stem + ".docx")).write_bytes(b"converted") + return MagicMock(returncode=0) + + mock_run.side_effect = run + @patch("chunksilo.docx_utils._get_doc_temp_dir") @patch("shutil.which", return_value=None) def test_libreoffice_not_found(self, mock_which, mock_temp_dir): @@ -93,15 +112,53 @@ def test_libreoffice_not_found(self, mock_which, mock_temp_dir): @patch("shutil.which") def test_conversion_success(self, mock_which, mock_run, mock_temp_dir, tmp_path): mock_which.side_effect = lambda p: "/usr/bin/soffice" if p == "/usr/bin/soffice" else None - mock_run.return_value = MagicMock(returncode=0) - mock_temp_dir.return_value = tmp_path + self._fake_soffice(mock_run) + temp_root = tmp_path / "doc_temp" + temp_root.mkdir() + mock_temp_dir.return_value = temp_root + source = tmp_path / "src" / "test.doc" + source.parent.mkdir() + source.write_bytes(b"legacy doc") + + result = _convert_doc_to_docx(source) + + assert result is not None and result.name == "test.docx" + assert result.read_bytes() == b"converted" + # Converted inside a private work dir under the storage temp root + assert result.parent.parent == temp_root + # ...from a copy of the source, not the source itself + argv = mock_run.call_args[0][0] + assert str(source) not in argv + assert str(result.parent / "test.doc") in argv + # ...with a private LibreOffice profile inside the work dir + assert any(a.startswith("-env:UserInstallation=") for a in argv) - # Create the expected output file - output_file = tmp_path / "test.docx" - output_file.touch() - - result = _convert_doc_to_docx(Path("/fake/test.doc")) - assert result == output_file + @patch("chunksilo.docx_utils._get_doc_temp_dir") + @patch("subprocess.run") + @patch("shutil.which") + def test_parallel_same_stem_sources_do_not_collide( + self, mock_which, mock_run, mock_temp_dir, tmp_path + ): + mock_which.side_effect = lambda p: "/usr/bin/soffice" if p == "/usr/bin/soffice" else None + self._fake_soffice(mock_run) + temp_root = tmp_path / "doc_temp" + temp_root.mkdir() + mock_temp_dir.return_value = temp_root + source_a = tmp_path / "a" / "report.doc" + source_b = tmp_path / "b" / "report.doc" + for src, content in ((source_a, b"content A"), (source_b, b"content B")): + src.parent.mkdir() + src.write_bytes(content) + + result_a = _convert_doc_to_docx(source_a) + result_b = _convert_doc_to_docx(source_b) + + assert result_a is not None and result_b is not None + assert result_a != result_b + assert result_a.parent != result_b.parent + # Each work dir holds the copy it converted, untouched by the other + assert (result_a.parent / "report.doc").read_bytes() == b"content A" + assert (result_b.parent / "report.doc").read_bytes() == b"content B" @patch("chunksilo.docx_utils._get_doc_temp_dir") @patch("subprocess.run") @@ -110,9 +167,13 @@ def test_conversion_timeout(self, mock_which, mock_run, mock_temp_dir, tmp_path) mock_which.side_effect = lambda p: "soffice" if p == "soffice" else None mock_run.side_effect = subprocess.TimeoutExpired(cmd="soffice", timeout=60) mock_temp_dir.return_value = tmp_path + source = tmp_path / "test.doc" + source.write_bytes(b"legacy doc") - result = _convert_doc_to_docx(Path("/fake/test.doc")) + result = _convert_doc_to_docx(source) assert result is None + # The failed conversion's work dir is cleaned up + assert list(tmp_path.iterdir()) == [source] @patch("chunksilo.docx_utils._get_doc_temp_dir") @patch("subprocess.run") @@ -121,9 +182,12 @@ def test_conversion_nonzero_exit(self, mock_which, mock_run, mock_temp_dir, tmp_ mock_which.side_effect = lambda p: "/usr/bin/soffice" if p == "/usr/bin/soffice" else None mock_run.return_value = MagicMock(returncode=1, stderr=b"error") mock_temp_dir.return_value = tmp_path + source = tmp_path / "test.doc" + source.write_bytes(b"legacy doc") - result = _convert_doc_to_docx(Path("/fake/test.doc")) + result = _convert_doc_to_docx(source) assert result is None + assert list(tmp_path.iterdir()) == [source] # ============================================================================= @@ -229,6 +293,20 @@ def test_heading_store_receives_headings(self, tmp_path): assert headings[1]["text"] == "H2" assert headings[1]["level"] == 2 + def test_heading_store_key_overrides_the_parsed_path(self, tmp_path): + """A converted .doc is parsed from a temp copy but looked up at query + time under the original path; the caller's key must win.""" + docx_path = _create_docx([("Heading 1", "H1"), ("Normal", "Body.")], tmp_path) + + mock_store = MagicMock() + split_docx_into_heading_documents( + docx_path, + heading_store=mock_store, + heading_store_key="/data/original.doc", + ) + + assert mock_store.set_headings.call_args[0][0] == "/data/original.doc" + def test_excluded_metadata_keys_passed_through(self, tmp_path): docx_path = _create_docx([ ("Heading 1", "Test"), diff --git a/test/test_never_mutates.py b/test/test_never_mutates.py new file mode 100644 index 0000000..ad653ba --- /dev/null +++ b/test/test_never_mutates.py @@ -0,0 +1,154 @@ +"""Regression guarantee: indexing NEVER writes inside an indexed directory. + +The indexed directories are other people's documents; ChunkSilo must be a pure +reader of them. This test builds an index over a populated source tree and +proves the tree is byte-for-byte identical afterwards - no lock files, no +converted artifacts, no logs, no renames, nothing added or removed. +""" +import copy +import hashlib +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from chunksilo import cfgload, index +from chunksilo.index import DirectoryConfig, IndexConfig, build_index + + +def _create_mock_embedding(): + from llama_index.core.embeddings import BaseEmbedding + + class MockEmbedding(BaseEmbedding): + def _get_vector(self, text: str): + vec = [0.0] * 384 + for word in text.lower().split(): + vec[sum(ord(c) for c in word) % 384] += 1.0 + return vec + + def _get_query_embedding(self, query: str): + return self._get_vector(query) + + def _get_text_embedding(self, text: str): + return self._get_vector(text) + + async def _aget_query_embedding(self, query: str): + return self._get_vector(query) + + return MockEmbedding(model_name="mock-test") + + +def _populate(data_dir: Path) -> None: + (data_dir / "notes.txt").write_text("plain text notes about the system") + (data_dir / "guide.md").write_text("# Guide\n\nHow to configure the thing.\n") + sub = data_dir / "sub dir with spaces" + sub.mkdir() + (sub / "deep.txt").write_text("content in a subdirectory") + + from docx import Document + + doc = Document() + doc.add_paragraph("Heading text", style="Heading 1") + doc.add_paragraph("Body of the docx document.") + doc.save(str(data_dir / "report.docx")) + + # A legacy .doc as well: without LibreOffice it is skipped with a warning, + # with LibreOffice it exercises the copy-first conversion path. Either way + # the source tree must come through untouched. + (data_dir / "legacy.doc").write_bytes(b"\xd0\xcf\x11\xe0 legacy doc bytes") + + +def _snapshot(root: Path) -> dict: + """Every path under root, with content hash + size + mtime for files.""" + entries = {} + for current, dirs, files in os.walk(root, followlinks=False): + for name in dirs: + path = Path(current) / name + entries[str(path.relative_to(root))] = ("dir",) + for name in files: + path = Path(current) / name + stat = path.lstat() + entries[str(path.relative_to(root))] = ( + "file", + stat.st_size, + stat.st_mtime_ns, + hashlib.sha256(path.read_bytes()).hexdigest(), + ) + return entries + + +@pytest.fixture +def indexing_env(tmp_path, monkeypatch): + data_dir = tmp_path / "source" + data_dir.mkdir() + storage_dir = tmp_path / "storage" + + # Run from a neutral CWD: anything that writes a relative path (a log, a + # scratch file) must land here or in storage, never in the source tree. + cwd = tmp_path / "cwd" + cwd.mkdir() + monkeypatch.chdir(cwd) + + test_index_config = IndexConfig( + directories=[DirectoryConfig(path=data_dir)], chunk_size=512, chunk_overlap=100 + ) + + # The active config governs cfgload.get() consumers (doc_temp location). + active = copy.deepcopy(cfgload._DEFAULTS) + active["storage"]["storage_dir"] = str(storage_dir) + monkeypatch.setattr(cfgload, "_config_cache", active) + monkeypatch.setattr(cfgload, "_active_path", tmp_path / "nonexistent.yaml") + + orig = ( + index.STORAGE_DIR, + index.STATE_DB_PATH, + index.BM25_INDEX_DIR, + index.HEADING_STORE_PATH, + index._config, + ) + index.STORAGE_DIR = storage_dir + index.STATE_DB_PATH = storage_dir / "ingestion_state.db" + index.BM25_INDEX_DIR = storage_dir / "bm25_index" + index.HEADING_STORE_PATH = storage_dir / "heading_store.json" + index._heading_store = None + + mock_embed = _create_mock_embedding() + with patch("chunksilo.index.load_index_config", return_value=test_index_config), \ + patch("chunksilo.index._create_fastembed_embedding", return_value=mock_embed), \ + patch("chunksilo.index.ensure_embedding_model_cached"), \ + patch("chunksilo.index.ensure_rerank_model_cached"): + yield data_dir + + ( + index.STORAGE_DIR, + index.STATE_DB_PATH, + index.BM25_INDEX_DIR, + index.HEADING_STORE_PATH, + index._config, + ) = orig + index._heading_store = None + + +def test_indexing_never_mutates_the_source_tree(indexing_env): + data_dir = indexing_env + _populate(data_dir) + + before = _snapshot(data_dir) + assert before, "populate created nothing?" + + build_index() # initial build + build_index() # incremental rebuild over the same content + + after = _snapshot(data_dir) + + assert set(after) == set(before), ( + "indexing added or removed entries in the source tree: " + f"added={set(after) - set(before)}, removed={set(before) - set(after)}" + ) + for rel, entry in before.items(): + assert after[rel] == entry, f"indexing modified {rel}: {entry} -> {after[rel]}" + + # And nothing wrote a log into the tree under any name. + assert not list(data_dir.rglob("*.log")) + assert not list(data_dir.rglob(".~lock*")) From 65cc38d5a42f55013297e4c0a405e2d0da951eee Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:02:04 +0200 Subject: [PATCH 04/18] fix: stop re-indexing untouched files and withhold deletions the scan could not verify Untouched files were re-indexed on every run, by three separate routes. A file that extracted no text never got a state row - load_files_parallel dropped it and build_index only recorded non-empty results - while its old chunks had already been deleted, so it looked new for ever; an image-only .docx hit this every time. A directory whose availability check was merely slow was dropped from the source list, and every file under it then looked deleted, wiping the tree from the index and forcing a full rebuild next run. A single file that could not be stat'd or hashed was pruned the same way, and once pruned it lost the mtime fast path, so the next scan had to hash it again - self-sustaining. Now every attempted file gets an outcome: an empty load is a final answer and is recorded with no doc ids; a failure is not recorded and is retried. Each source tracks the health of its scan, deletions are withheld for anything a scan could not see (with the count reported and the reason logged), and each reprocessed file logs why. The scan now runs in two phases - a stat-only enumeration, then hashing - with reason-returning variants of the include/exclude matchers and an optional per-event callback, so diagnostics can reuse the exact scan the indexer performs. This also fixes a pre-existing race: directory pruning ran on the consumer side of the walk while the os.walk producer thread walked ahead, so excluded directories were often descended into anyway. Pruning now happens inside the walker thread before descent. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- src/chunksilo/index.py | 482 +++++++++++++++++++++++++++----- test/test_incremental_ingest.py | 200 +++++++++++++ 2 files changed, 616 insertions(+), 66 deletions(-) diff --git a/src/chunksilo/index.py b/src/chunksilo/index.py index a3d5bba..68def08 100644 --- a/src/chunksilo/index.py +++ b/src/chunksilo/index.py @@ -14,7 +14,7 @@ import threading import time from abc import ABC, abstractmethod -from collections.abc import Iterator +from collections.abc import Callable, Iterator from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path @@ -280,6 +280,53 @@ class FileInfo: source_dir: str = "" # Tracks which configured directory this file came from +@dataclass +class FileCandidate: + """A discovered file before hashing. + + Revision grouping needs every file's mtime before it can decide which + files to index at all, and hashing is far too expensive to spend on + files that grouping will then discard - they have no state row, so no + mtime fast path saves them. + Hence the scan runs in two phases: stat-only candidates first, full + FileInfo only for the keepers. + """ + path: str + mtime: float + source_dir: str = "" + + +@dataclass +class ScanEvent: + """A file or directory the scan saw but will not index. Diagnostic only.""" + + kind: str # "excluded_file" | "pruned_dir" | "unreadable_file" + path: str # absolute + pattern: str | None = None # deciding pattern, when one exists + + +@dataclass +class ScanStatus: + """Health of one directory scan. + + A file missing from a scan means one of two very different things: it was + deleted, or the scan could not look. Pruning on the second is destructive - + the file's chunks are dropped from the index and its state row removed, so + the next run re-indexes it from scratch, and a laggy network mount turns + that into a permanent re-index loop. Deletions are therefore withheld + whenever a scan was not clean. + """ + + complete: bool = True + reason: str = "" + unreadable: set[str] = field(default_factory=set) + + def fail(self, reason: str) -> None: + self.complete = False + if not self.reason: + self.reason = reason + + class IngestionState: """Manages the state of ingested files using a SQLite database.""" @@ -421,6 +468,27 @@ def load_file( """ pass + def unscanned_roots(self) -> dict[str, str]: + """Directory roots not fully enumerated by the last scan -> reason.""" + return {} + + def unreadable_paths(self) -> set[str]: + """Paths the last scan could not stat or hash.""" + return set() + + +def _owning_root(path: str, roots: dict[str, str]) -> str | None: + """The root in ``roots`` that contains ``path``, or None. + + Plain prefix matching on the absolute paths the state DB stores; a trailing + separator is required so ``/data`` does not claim ``/data-archive``. + """ + for root in roots: + if path == root or path.startswith(root.rstrip(os.sep) + os.sep): + return root + return None + + def _compute_line_offsets(text: str) -> list[int]: """Compute character offset positions for each line start. @@ -596,11 +664,14 @@ class LocalFileSystemSource(DataSource): def __init__(self, config: DirectoryConfig): self.config = config self.base_dir = config.path + self.scan_status = ScanStatus() def is_available(self) -> bool: """Check if the directory is available and accessible. Runs with a timeout to avoid hanging on unresponsive network mounts. + A timeout is recorded on ``scan_status`` so callers can tell it apart + from a directory that is genuinely gone. """ timeout = cfgload.get("indexing.scan_item_seconds") @@ -613,23 +684,38 @@ def _check(): # Try to list directory to verify access (important for network mounts) next(self.base_dir.iterdir(), None) return True - except (OSError, PermissionError): - return False - - result = _run_with_timeout(_check, timeout_seconds=timeout, default=False) - if result is False and timeout > 0: - # Distinguish genuine "not a dir" from timeout — log only for timeout - # (the sentinel default=False means we can't distinguish here, but - # _run_with_timeout already logged nothing; let callers log.) - pass - return result + except (OSError, PermissionError) as exc: + return exc + + result = _run_with_timeout(_check, timeout_seconds=timeout) + if result is _SCAN_TIMEOUT_SENTINEL: + logger.warning( + "Availability check for %s timed out after %ss", self.base_dir, timeout + ) + self.scan_status.fail(f"availability check timed out after {timeout}s") + return False + if isinstance(result, BaseException): + logger.warning("Cannot access %s: %s", self.base_dir, result) + self.scan_status.fail(f"not accessible: {result}") + return False + return bool(result) def _matches_patterns(self, file_path: Path) -> bool: - """Check if file matches include patterns and doesn't match exclude patterns. + """Check if file matches include patterns and doesn't match exclude patterns.""" + return self._match_decision(file_path)[0] + + def _match_decision(self, file_path: Path) -> tuple[bool, str | None]: + """Match verdict plus the pattern that decided it. Uses PurePosixPath.match() for glob pattern matching. For directory exclusion patterns like **/*venv*/**, checks each path component. When case_sensitive is False (default), matching is case-insensitive. + + Returns: + (False, pattern) - excluded by that pattern (as written in config) + (True, pattern) - matched that include pattern + (True, None) - no include patterns configured, everything passes + (False, None) - no include pattern matched """ import fnmatch from pathlib import PurePosixPath @@ -654,27 +740,33 @@ def _matches_patterns(self, file_path: Path) -> bool: dir_pattern = pat[3:-3] # Remove **/ prefix and /** suffix for part in rel_path.parts[:-1]: # Check all directory components (not filename) if fnmatch.fnmatch(part.lower() if ci else part, dir_pattern): - return False + return False, pattern else: # Standard pattern matching if PurePosixPath(rel_str).match(pat) or name == pat: - return False + return False, pattern # Check include patterns if not self.config.include: - return True + return True, None for pattern in self.config.include: pat = pattern.lower() if ci else pattern if PurePosixPath(rel_str).match(pat) or PurePosixPath(abs_str).match(pat): - return True + return True, pattern - return False + return False, None def _should_skip_directory(self, dir_name: str) -> bool: - """Check if a directory should be pruned from os.walk traversal. + """Check if a directory should be pruned from os.walk traversal.""" + return self._directory_skip_pattern(dir_name) is not None + + def _directory_skip_pattern(self, dir_name: str) -> str | None: + """The exclude pattern that prunes this directory from os.walk, or None. - Handles patterns of the form **//** by checking the directory name. + Handles patterns of the form **//** by checking the directory + name. Unlike file matching, this has always been case-sensitive + regardless of the case_sensitive setting. """ import fnmatch @@ -682,15 +774,23 @@ def _should_skip_directory(self, dir_name: str) -> bool: if pattern.startswith('**/') and pattern.endswith('/**'): dir_pattern = pattern[3:-3] # e.g., "node_modules", ".git", "*venv*" if fnmatch.fnmatch(dir_name, dir_pattern): - return True - return False + return pattern + return None - def _walk_with_timeout(self): + def _walk_with_timeout( + self, on_event: Callable[["ScanEvent"], None] | None = None + ): """Yield (root, dirs, files) tuples from os.walk with per-iteration timeout. Runs os.walk in a daemon thread, feeding results through a queue. + Excluded directories are pruned inside the walker thread, before + os.walk descends - pruning from the consumer side would race the + producer, which walks ahead of consumption. If no result arrives within scan_item_seconds, the walk is considered stalled and iteration stops. + + on_event, when given, receives a ScanEvent per pruned directory. It + may be invoked from the walker thread. """ timeout = cfgload.get("indexing.scan_item_seconds") q: queue.Queue = queue.Queue() @@ -698,8 +798,21 @@ def _walk_with_timeout(self): def _producer(): try: - for entry in os.walk(self.base_dir): - q.put(entry) + for root, dirs, files in os.walk(self.base_dir): + kept_dirs = [] + for d in dirs: + skip_pattern = self._directory_skip_pattern(d) + if skip_pattern is None: + kept_dirs.append(d) + elif on_event is not None: + on_event(ScanEvent( + "pruned_dir", + str((Path(root) / d).absolute()), + skip_pattern, + )) + # Prune in-place to prevent descent + dirs[:] = kept_dirs + q.put((root, dirs, files)) q.put(_sentinel) except Exception as exc: q.put(exc) @@ -715,6 +828,9 @@ def _producer(): f"os.walk() stalled for {timeout}s on {self.base_dir}, " "aborting directory scan" ) + # The rest of the tree was never visited: anything already + # indexed under it must not be mistaken for deleted. + self.scan_status.fail(f"directory walk stalled after {timeout}s") return if item is _sentinel: return @@ -729,21 +845,45 @@ def iter_files(self, tracked_files: dict[str, dict] | None = None) -> Iterator[F tracked_files: Optional dict of previously tracked file states, keyed by absolute path. Used for mtime-based fast pre-check. """ - if self.config.recursive: - for root, dirs, files in self._walk_with_timeout(): - # Prune excluded directories in-place to prevent descent - dirs[:] = [d for d in dirs if not self._should_skip_directory(d)] + for candidate in self.iter_candidates(): + try: + yield self.file_info_for(candidate, tracked_files) + except (OSError, TimeoutError): + continue + + def iter_candidates( + self, *, on_event: Callable[[ScanEvent], None] | None = None + ) -> Iterator[FileCandidate]: + """Yield a stat-only FileCandidate for each matching file. + + Resets scan_status; file_info_for failures accumulate into the same + status afterwards, so together the two phases report one scan. + on_event, when given, receives a ScanEvent for every file or directory + the scan saw but skipped. Diagnostics only; the indexing path never + sets it. + """ + self.scan_status = ScanStatus() + + if self.config.recursive: + for root, dirs, files in self._walk_with_timeout(on_event=on_event): root_path = Path(root) for file in files: file_path = root_path / file - if not self._matches_patterns(file_path): - continue - try: - yield self._create_file_info(file_path, tracked_files) - except (OSError, TimeoutError) as e: - logger.warning(f"Could not access file {file_path}: {e}") + matched, pattern = self._match_decision(file_path) + if not matched: + if on_event is not None: + on_event(ScanEvent( + "excluded_file", str(file_path.absolute()), pattern + )) continue + candidate = self._stat_candidate(file_path) + if candidate is not None: + yield candidate + elif on_event is not None: + on_event(ScanEvent( + "unreadable_file", str(file_path.absolute()) + )) else: # Non-recursive: only top-level files try: @@ -752,16 +892,63 @@ def iter_files(self, tracked_files: dict[str, dict] | None = None) -> Iterator[F ] except OSError as e: logger.warning(f"Could not list directory {self.base_dir}: {e}") + self.scan_status.fail(f"could not be listed: {e}") return for f in top_files: - if not self._matches_patterns(f): - continue - try: - yield self._create_file_info(f, tracked_files) - except (OSError, TimeoutError) as e: - logger.warning(f"Could not access file {f}: {e}") + matched, pattern = self._match_decision(f) + if not matched: + if on_event is not None: + on_event(ScanEvent( + "excluded_file", str(f.absolute()), pattern + )) continue + candidate = self._stat_candidate(f) + if candidate is not None: + yield candidate + elif on_event is not None: + on_event(ScanEvent("unreadable_file", str(f.absolute()))) + + def _stat_candidate(self, file_path: Path) -> FileCandidate | None: + """Stat a file with timeout protection; None marks it unreadable.""" + timeout = cfgload.get("indexing.scan_item_seconds") + try: + mtime = _run_with_timeout( + lambda: file_path.stat().st_mtime, timeout_seconds=timeout + ) + except OSError as e: + logger.warning(f"Could not access file {file_path}: {e}") + self.scan_status.unreadable.add(str(file_path.absolute())) + return None + if mtime is _SCAN_TIMEOUT_SENTINEL: + logger.warning( + f"Timed out after {timeout}s accessing file {file_path}, skipping" + ) + self.scan_status.unreadable.add(str(file_path.absolute())) + return None + return FileCandidate( + path=str(file_path.absolute()), + mtime=mtime, + source_dir=str(self.base_dir.absolute()), + ) + + def file_info_for( + self, + candidate: FileCandidate, + tracked_files: dict[str, dict] | None = None, + ) -> FileInfo: + """Full FileInfo (with content hash) for a candidate. + + Records the file as unreadable and re-raises on failure, so a file + that vanishes or stalls between the two scan phases is withheld from + deletion exactly as if the walk had not seen it. + """ + try: + return self._create_file_info(Path(candidate.path), tracked_files) + except (OSError, TimeoutError) as e: + logger.warning(f"Could not access file {candidate.path}: {e}") + self.scan_status.unreadable.add(candidate.path) + raise def _create_file_info( self, @@ -826,6 +1013,14 @@ def _create_file_info_inner( source_dir=str(self.base_dir.absolute()), ) + def unscanned_roots(self) -> dict[str, str]: + if self.scan_status.complete: + return {} + return {str(self.base_dir.absolute()): self.scan_status.reason} + + def unreadable_paths(self) -> set[str]: + return set(self.scan_status.unreadable) + def load_file( self, file_info: FileInfo, @@ -983,6 +1178,9 @@ def __init__(self, config: IndexConfig): self.config = config self.sources: list[LocalFileSystemSource] = [] self.unavailable_dirs: list[DirectoryConfig] = [] + # Roots whose contents were not fully enumerated this run, with the + # reason. Deletions under them are withheld (see unscanned_roots). + self._unavailable_reasons: dict[str, str] = {} for dir_config in config.directories: if not dir_config.enabled: @@ -996,18 +1194,65 @@ def __init__(self, config: IndexConfig): logger.info(f"Added directory source: {dir_config.path}") else: self.unavailable_dirs.append(dir_config) + self._unavailable_reasons[str(dir_config.path.absolute())] = ( + source.scan_status.reason or "directory unavailable" + ) logger.warning(f"Directory unavailable, skipping: {dir_config.path}") def iter_files(self, tracked_files: dict[str, dict] | None = None) -> Iterator[FileInfo]: """Iterate over files from all available sources.""" + for candidate in self.iter_candidates(): + try: + yield self.file_info_for(candidate, tracked_files) + except (OSError, TimeoutError): + continue + + def iter_candidates(self) -> Iterator[FileCandidate]: + """Stat-only candidates from all available sources.""" seen_paths: set[str] = set() for source in self.sources: - for file_info in source.iter_files(tracked_files=tracked_files): + for candidate in source.iter_candidates(): # Deduplicate in case of overlapping mounts - if file_info.path not in seen_paths: - seen_paths.add(file_info.path) - yield file_info + if candidate.path not in seen_paths: + seen_paths.add(candidate.path) + yield candidate + + def file_info_for( + self, + candidate: FileCandidate, + tracked_files: dict[str, dict] | None = None, + ) -> FileInfo: + """Full FileInfo via the source that owns the candidate.""" + for source in self.sources: + if candidate.source_dir == str(source.base_dir.absolute()): + return source.file_info_for(candidate, tracked_files) + + # Fallback: use first source (shouldn't happen normally) + if self.sources: + return self.sources[0].file_info_for(candidate, tracked_files) + + raise ValueError(f"No source available for file: {candidate.path}") + + def unscanned_roots(self) -> dict[str, str]: + """Directory roots that were not fully enumerated, mapped to the reason. + + A root lands here when it was unavailable at startup or when its walk + stalled or could not be listed. Callers must not prune tracked files + under these roots. + """ + roots = dict(self._unavailable_reasons) + for source in self.sources: + if not source.scan_status.complete: + roots[str(source.base_dir.absolute())] = source.scan_status.reason + return roots + + def unreadable_paths(self) -> set[str]: + """Files that exist but could not be stat'd or hashed during this scan.""" + paths: set[str] = set() + for source in self.sources: + paths |= source.scan_status.unreadable + return paths def load_file( self, @@ -1303,6 +1548,22 @@ def calculate_optimal_batch_size( return min(batch_size, num_files) +@dataclass +class LoadOutcome: + """Result of attempting to load one file. + + ``failed`` separates "extracted nothing" from "could not be read". An empty + but successful load is a final answer and gets a state row (with no doc ids) + so the file is not retried until it changes; a failure gets none, so the + next run tries again. + """ + + file_info: FileInfo + docs: list[Any] = field(default_factory=list) + failed: bool = False + + + def load_files_parallel( files: list[FileInfo], data_source: DataSource, @@ -1312,7 +1573,7 @@ def load_files_parallel( per_file_timeout: float = 300, heartbeat_interval: float = 2.0, abort_ctl: "GracefulAbort | None" = None, -) -> dict[str, tuple[FileInfo, list[Any]]]: +) -> dict[str, LoadOutcome]: """Load files in parallel using ThreadPoolExecutor. Args: @@ -1326,11 +1587,13 @@ def load_files_parallel( abort_ctl: Optional GracefulAbort instance for Ctrl-C handling Returns: - Dict mapping file path to (FileInfo, list[LlamaIndexDocument]) + Dict mapping file path to :class:`LoadOutcome`. Every attempted file is + present, including ones that extracted no text and ones that failed - + the caller needs both to decide whether to record state. """ - file_docs = {} + file_docs: dict[str, LoadOutcome] = {} - def load_single_file(file_info: FileInfo): + def load_single_file(file_info: FileInfo) -> LoadOutcome: """Load a single file (executed in thread pool).""" try: with FileProcessingContext( @@ -1341,13 +1604,13 @@ def load_single_file(file_info: FileInfo): ) as ctx: ctx.set_phase("Loading file") docs = data_source.load_file(file_info, ctx) - return file_info, docs + return LoadOutcome(file_info=file_info, docs=docs) except FileProcessingTimeoutError as e: logger.error(f"Timeout loading {file_info.path}: {e}") - return file_info, [] + return LoadOutcome(file_info=file_info, failed=True) except Exception as e: logger.error(f"Error loading {file_info.path}: {e}") - return file_info, [] + return LoadOutcome(file_info=file_info, failed=True) with ThreadPoolExecutor(max_workers=max_workers) as executor: # Submit all file loading tasks @@ -1365,11 +1628,11 @@ def load_single_file(file_info: FileInfo): file_info = future_to_file[future] try: - result_file_info, docs = future.result(timeout=per_file_timeout + 10) - if docs: # Only add if we got documents - file_docs[result_file_info.path] = (result_file_info, docs) + outcome = future.result(timeout=per_file_timeout + 10) + file_docs[outcome.file_info.path] = outcome except Exception as e: logger.error(f"Error retrieving result for {file_info.path}: {e}") + file_docs[file_info.path] = LoadOutcome(file_info=file_info, failed=True) finally: ui.progress_update() @@ -1480,22 +1743,57 @@ def build_index( new_count = 0 modified_count = 0 + # Phase 1: enumerate cheaply (stat only). Hashing happens in phase 2, + # and a file that fails either phase is recorded as unreadable so that + # its deletion is withheld below. ui.step_start("Scanning for changes") scan_count = 0 - for file_info in data_source.iter_files(tracked_files=tracked_files): + candidates: list[FileCandidate] = [] + for candidate in data_source.iter_candidates(): if abort_ctl.abort_requested: break scan_count += 1 if scan_count % 100 == 0: ui.step_update(f"Scanning for changes ({scan_count:,} files)") + candidates.append(candidate) + + if abort_ctl.abort_requested: + ui.step_done("interrupted") + abort_ctl.uninstall() + ui.print(f"{ui.YELLOW}Interrupted during scan. No changes made.{ui.RESET}") + return + + # Phase 2: hash the candidates. + hashed_count = 0 + for candidate in candidates: + if abort_ctl.abort_requested: + break + hashed_count += 1 + if hashed_count % 100 == 0: + ui.step_update( + f"Scanning for changes ({hashed_count:,}/{len(candidates):,} files)" + ) + try: + file_info = data_source.file_info_for(candidate, tracked_files) + except (OSError, TimeoutError): + continue # recorded as unreadable; its deletion is withheld found_files.add(file_info.path) existing_state = tracked_files.get(file_info.path) if existing_state: + # Say why. A file that keeps reappearing here with no reason a + # human recognises is the signature of an indexing bug, and + # without this line there is nothing to go on. if existing_state["hash"] != file_info.hash: + reason = "content changed" + else: + reason = "" + if reason: + logger.info("Reprocessing %s: %s", file_info.path, reason) modified_count += 1 files_to_process.append(file_info) else: + logger.debug("Indexing new file %s", file_info.path) new_count += 1 files_to_process.append(file_info) @@ -1505,8 +1803,30 @@ def build_index( ui.print(f"{ui.YELLOW}Interrupted during scan. No changes made.{ui.RESET}") return - # Identify Deleted Files - deleted_files = set(tracked_files.keys()) - found_files + # Identify Deleted Files. A path missing from the scan is only deleted + # if the scan could actually see where it lives; otherwise a stalled + # mount would wipe the whole tree from the index and force a full + # re-index on the next run. + missing = set(tracked_files.keys()) - found_files + unreadable = data_source.unreadable_paths() + unscanned = data_source.unscanned_roots() + withheld: dict[str, str] = {} + for path in missing: + if path in unreadable: + withheld[path] = "could not be read during this scan" + continue + root = _owning_root(path, unscanned) + if root is not None: + withheld[path] = f"{root} was not fully scanned ({unscanned[root]})" + deleted_files = missing - withheld.keys() + + if withheld: + reasons = sorted(set(withheld.values())) + logger.warning( + "Keeping %d indexed file(s) that this scan did not see: %s", + len(withheld), + "; ".join(reasons), + ) # Build summary suffix parts = [] @@ -1516,6 +1836,8 @@ def build_index( parts.append(f"{modified_count} modified") if deleted_files: parts.append(f"{len(deleted_files)} deleted") + if withheld: + parts.append(f"{len(withheld)} unseen (kept)") ui.step_done(", ".join(parts) if parts else "no changes") # Process deletions @@ -1534,6 +1856,8 @@ def build_index( return # Process New/Modified Files + empty_files: list[str] = [] + failed_files: list[str] = [] if files_to_process: # Get configuration (0 disables the per-file timeout) per_file_timeout = cfgload.get("indexing.per_file_seconds") @@ -1586,7 +1910,6 @@ def build_index( # Load files in this batch accumulated_docs = [] - doc_to_file_mapping = {} file_doc_ids = {} workers = max_workers @@ -1601,11 +1924,10 @@ def build_index( abort_ctl=abort_ctl, ) - for file_path, (file_info, docs) in file_docs.items(): + for file_path, outcome in file_docs.items(): file_doc_ids[file_path] = [] - for doc in docs: + for doc in outcome.docs: accumulated_docs.append(doc) - doc_to_file_mapping[doc.doc_id] = file_info file_doc_ids[file_path].append(doc.doc_id) # Batch embedding @@ -1631,12 +1953,20 @@ def build_index( index.insert_nodes(nodes) ui.progress_substep_done() - # Batch update state + # Batch update state. A file that yielded no documents still + # gets a row (with no doc ids): that is a final answer for this + # content, and skipping it would make the file look new on every + # subsequent run and be reprocessed forever. Failures get no row + # so that they are retried. state_updates = [] - for file_path, doc_ids in file_doc_ids.items(): - if doc_ids: - file_info = doc_to_file_mapping[doc_ids[0]] - state_updates.append((file_info, doc_ids)) + for file_path, outcome in file_docs.items(): + if outcome.failed: + failed_files.append(file_path) + continue + doc_ids = file_doc_ids.get(file_path, []) + if not doc_ids: + empty_files.append(file_path) + state_updates.append((outcome.file_info, doc_ids)) if state_updates: ingestion_state.update_file_states_batch(state_updates) @@ -1689,3 +2019,23 @@ def build_index( abort_ctl.uninstall() ui.success(f"\nIndexing complete. {len(files_to_process)} files processed.") + if empty_files: + logger.info("Files that yielded no text: %s", ", ".join(empty_files[:20])) + ui.print( + f"{ui.YELLOW}{len(empty_files)} file(s) contained no extractable " + f"text and were recorded as empty; they will not be retried until " + f"they change. Re-run with --verbose to list them.{ui.RESET}" + ) + if failed_files: + logger.warning("Files that failed to load: %s", ", ".join(failed_files[:20])) + ui.print( + f"{ui.YELLOW}{len(failed_files)} file(s) could not be read and will " + f"be retried on the next run. Re-run with --verbose to list them.{ui.RESET}" + ) + if withheld: + ui.print( + f"{ui.YELLOW}{len(withheld)} indexed file(s) were kept even though " + f"this scan did not see them, because the scan was incomplete. They " + f"were NOT removed from the index. Re-run with --verbose to see why.{ui.RESET}" + ) + diff --git a/test/test_incremental_ingest.py b/test/test_incremental_ingest.py index 5a50c2f..b03ac74 100644 --- a/test/test_incremental_ingest.py +++ b/test/test_incremental_ingest.py @@ -1,3 +1,4 @@ +import contextlib import os import shutil import sqlite3 @@ -126,3 +127,202 @@ def test_incremental_ingestion(test_env): (data_dir / "doc2.txt").unlink() build_index() check_db_count(db_path, 1) + + +# =========================================================================== +# Files must not be reprocessed on every run +# +# Three ways a build used to lose a file's state row and re-index it forever: +# it extracted no text, its directory was unavailable, or it could not be +# stat'd. All three showed up in production as "untouched files keep being +# re-indexed". +# =========================================================================== + +class _LogSink(logging.Handler): + """Collects records from chunksilo.index. + + Not a StreamHandler on purpose: IndexingUI mutes root StreamHandlers while + it owns the terminal, which silences pytest's caplog for the whole build. + """ + + def __init__(self): + super().__init__(level=logging.DEBUG) + self.messages: list[str] = [] + + def emit(self, record): + self.messages.append(record.getMessage()) + + @property + def text(self) -> str: + return "\n".join(self.messages) + + +@contextlib.contextmanager +def capture_index_log(): + sink = _LogSink() + log = logging.getLogger("chunksilo.index") + previous = log.level + log.setLevel(logging.DEBUG) + log.addHandler(sink) + try: + yield sink + finally: + log.removeHandler(sink) + log.setLevel(previous) + + +def _tracked(db_path): + """{path: (hash, doc_ids)} from the state DB.""" + with sqlite3.connect(db_path) as conn: + return { + row[0]: (row[1], row[2]) + for row in conn.execute("SELECT path, hash, doc_ids FROM files") + } + + +def test_file_yielding_no_documents_is_recorded_once(test_env): + """A file that extracts no text gets a state row and is not retried. + + Without this it is dropped from the state DB every run, looks new on the + next one, and is reprocessed forever. + """ + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "empty.txt", "some text") + real_load = index.LocalFileSystemSource.load_file + + def load_nothing(self, file_info, ctx=None): + if file_info.path.endswith("empty.txt"): + return [] + return real_load(self, file_info, ctx) + + with patch.object(index.LocalFileSystemSource, "load_file", load_nothing): + build_index() + row = _tracked(db_path).get(str((data_dir / "empty.txt").absolute())) + assert row is not None, "empty file must still be tracked" + assert row[1] == "", "an empty load stores no doc ids" + + with capture_index_log() as log: + build_index() + assert "Reprocessing" not in log.text + assert "Indexing new file" not in log.text + + +def test_failed_file_is_retried(test_env): + """A file that could not be read gets no state row, so it is retried.""" + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "boom.txt", "some text") + + def load_boom(self, file_info, ctx=None): + raise OSError("mount went away") + + with patch.object(index.LocalFileSystemSource, "load_file", load_boom): + build_index() + assert str((data_dir / "boom.txt").absolute()) not in _tracked(db_path) + + +def test_unavailable_directory_does_not_delete_its_files(test_env): + """One unreachable directory must not wipe its files from the index. + + This is the destructive case: a laggy network mount used to prune the whole + tree, which forced a full re-index on the following run. (When *every* + directory is unavailable build_index bails out before touching anything; + the damage needed a mixed config, which is what this sets up.) + """ + data_dir, _storage_dir, db_path = test_env + other_dir = data_dir.parent / "other" + other_dir.mkdir() + create_file(data_dir, "doc1.txt", "This is document 1.") + create_file(other_dir, "doc2.txt", "This is document 2.") + + two_dirs = IndexConfig( + directories=[DirectoryConfig(path=data_dir), DirectoryConfig(path=other_dir)], + chunk_size=512, + chunk_overlap=100, + ) + real_is_available = index.LocalFileSystemSource.is_available + + def only_other_available(self): + if self.base_dir == data_dir: + self.scan_status.fail("availability check timed out after 30s") + return False + return real_is_available(self) + + with patch("chunksilo.index.load_index_config", return_value=two_dirs): + build_index() + assert len(_tracked(db_path)) == 2 + + with patch.object( + index.LocalFileSystemSource, "is_available", only_other_available + ), capture_index_log() as log: + build_index() + + assert len(_tracked(db_path)) == 2, "the mount was down, not the file" + assert "did not see" in log.text + + +def test_stalled_walk_does_not_delete_its_files(test_env): + """A directory walk that stalls halfway must not prune what it never reached.""" + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "doc1.txt", "This is document 1.") + create_file(data_dir, "doc2.txt", "This is document 2.") + build_index() + assert len(_tracked(db_path)) == 2 + + def stalled_walk(self, on_event=None): + # Yield nothing and report the scan as incomplete, as the real + # _walk_with_timeout does when os.walk() stops responding. + self.scan_status.fail("directory walk stalled after 30s") + return iter(()) + + with patch.object(index.LocalFileSystemSource, "_walk_with_timeout", stalled_walk): + build_index() + + assert len(_tracked(db_path)) == 2 + + +def test_unreadable_file_is_not_treated_as_deleted(test_env): + """A file whose stat/hash fails is kept, not pruned and re-added next run.""" + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "doc1.txt", "This is document 1.") + create_file(data_dir, "doc2.txt", "This is document 2.") + build_index() + before = _tracked(db_path) + assert len(before) == 2 + + real_create = index.LocalFileSystemSource._create_file_info + + def flaky_create(self, file_path, tracked_files=None): + if file_path.name == "doc2.txt": + raise OSError("stale file handle") + return real_create(self, file_path, tracked_files) + + with patch.object(index.LocalFileSystemSource, "_create_file_info", flaky_create): + build_index() + + assert _tracked(db_path) == before + + +def test_genuinely_deleted_file_is_still_pruned(test_env): + """Withholding deletions must not stop real deletions from being applied.""" + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "doc1.txt", "This is document 1.") + create_file(data_dir, "doc2.txt", "This is document 2.") + build_index() + assert len(_tracked(db_path)) == 2 + + (data_dir / "doc2.txt").unlink() + build_index() + assert len(_tracked(db_path)) == 1 + + +def test_reprocess_reason_is_logged(test_env): + """Every reprocessed file says why, so a re-index loop is diagnosable.""" + data_dir, _storage_dir, _db_path = test_env + create_file(data_dir, "doc1.txt", "This is document 1.") + build_index() + + create_file(data_dir, "doc1.txt", "This is document 1, edited.") + with capture_index_log() as log: + build_index() + assert "Reprocessing" in log.text + assert "content changed" in log.text From be55b46cd1b2e31426399a0e945331d34c8d0beb Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:08:36 +0200 Subject: [PATCH 05/18] feat: group document revisions and index only the newest Document trees often hold many ad-hoc revisions of one document (revision-named directories like .../PB3/, filename tokens like 'Spec PB3.docx' or 'Spec_v2.pdf') plus copies made to collect review comments, and every one of them was indexed. - A new revisions module groups files that are revisions of one document: by a configurable doc-ID regex first, else by revision-stripped filename and path. Only the newest member (by file mtime - labels are never ordered) is indexed; review copies are never indexed. Skipped files are pruned by the existing deletion loop and resurface automatically when they become newest again. No state-schema change. - The two-phase scan hashes only the keepers, so skipped files cost no MD5 work. - indexing.versioning.index_superseded keeps old revisions indexed, stamped superseded so the query side can demote them. - Chunks and BM25 file-name nodes carry doc_group / doc_id / superseded metadata for the query side. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- config.yaml | 12 ++ src/chunksilo/cfgload.py | 32 ++++ src/chunksilo/index.py | 88 +++++++++- src/chunksilo/revisions.py | 233 ++++++++++++++++++++++++++ test/test_revision_indexing.py | 294 +++++++++++++++++++++++++++++++++ test/test_revisions.py | 277 +++++++++++++++++++++++++++++++ 6 files changed, 929 insertions(+), 7 deletions(-) create mode 100644 src/chunksilo/revisions.py create mode 100644 test/test_revision_indexing.py create mode 100644 test/test_revisions.py diff --git a/config.yaml b/config.yaml index cf42b26..19533c3 100644 --- a/config.yaml +++ b/config.yaml @@ -39,6 +39,18 @@ indexing: ## Diagnostics # slow_file_threshold_seconds: 30 # warn when one file takes longer than this + ## Revision handling: group ad-hoc document revisions (".../PB3/Spec.docx", + ## "Spec PB3.docx", "Spec_v2.pdf") and index only the newest per group, + ## by file modification time. Review-comment copies are never indexed. + # versioning: + # enabled: true + # doc_id_patterns: [] # e.g. ['\b(DOC-\d{5})\b'] - files sharing an + # # extracted document ID group together + # review_copy_patterns: ['\breview\b'] # replaces the defaults (review, comments, + # # kommentarer, granskning, copy, kopia); [] disables + # extra_revision_tokens: [] # extra token regex fragments, e.g. ['draft\d+'] + # index_superseded: false # index older revisions too, demoted in results + # Confluence integration (optional) - enables live search of Confluence pages # Installed by default; set the values below to enable. confluence: diff --git a/src/chunksilo/cfgload.py b/src/chunksilo/cfgload.py index 2ac44ed..0703b78 100644 --- a/src/chunksilo/cfgload.py +++ b/src/chunksilo/cfgload.py @@ -67,6 +67,19 @@ def _find_config() -> Path: "**/.DS_Store", ] +# Filename markers of copies made to collect review comments, matched +# case-insensitively against the filename without its extension. Setting +# indexing.versioning.review_copy_patterns replaces this list wholesale; +# [] turns detection off. +DEFAULT_REVIEW_COPY_PATTERNS = [ + r"\breview\b", + r"\bcomments?\b", + r"\bkommentar(?:er)?\b", + r"\bgranskning\b", + r"\bcopy\b", # "Copy of Spec", "Spec - Copy (2)" + r"\bkopia\b", # "Kopia av Spec", "Spec - kopia" +] + _DEFAULTS: dict[str, Any] = { "indexing": { "directories": ["./data"], @@ -85,6 +98,25 @@ def _find_config() -> Path: "per_file_seconds": 300, # give up on a single file; 0 disables "scan_item_seconds": 30, # timeout for stat/hash/walk during scanning "slow_file_threshold_seconds": 30, # warn when one file takes longer + "versioning": { + # Group files that are revisions of one document (revision-named + # directories or filename tokens like "Spec PB3.docx") and index + # only the newest member of each group, by modification time. + "enabled": True, + # Regexes tried in order against the full path; files sharing an + # extracted document ID form one group. The first capture group + # (or the whole match) is the ID. Empty = group by normalized + # name/path only. + "doc_id_patterns": [], + # Extra revision-token regex fragments (case-insensitive), added + # to the built-in grammar for directory names and filename + # suffixes alike. + "extra_revision_tokens": [], + "review_copy_patterns": DEFAULT_REVIEW_COPY_PATTERNS, + # Index older revisions too, stamped superseded and ranked below + # current documents in results. + "index_superseded": False, + }, }, "retrieval": { "embed_model_name": "BAAI/bge-small-en-v1.5", diff --git a/src/chunksilo/index.py b/src/chunksilo/index.py index 68def08..2950224 100644 --- a/src/chunksilo/index.py +++ b/src/chunksilo/index.py @@ -35,7 +35,7 @@ from llama_index.embeddings.fastembed import FastEmbedEmbedding # Load configuration from config.yaml -from . import cfgload +from . import cfgload, revisions from .cfgload import DEFAULT_EXCLUDE_PATTERNS, DEFAULT_INCLUDE_PATTERNS, load_config from .docx_utils import _convert_doc_to_docx, cleanup_conversion_dir, split_docx_into_heading_documents from .models import _get_cached_model_path, configure_offline_mode, resolve_flashrank_model_name @@ -74,6 +74,9 @@ "last_modified_date",# temporal, not semantic "doc_ids", # internal tracking "hash", # internal tracking + "doc_group", # revision-group key, not semantic + "doc_id", # document ID, not semantic + "superseded", # revision status, not semantic ] # These keys are excluded from the LLM context to save context window @@ -83,6 +86,9 @@ "doc_ids", # internal tracking "file_path", # usually redundant if file_name is present "source", # usually redundant + "doc_group", # revision-group key, not for the LLM + "doc_id", # document ID, surfaced via result formatting instead + "superseded", # revision status, not for the LLM ] logger = logging.getLogger(__name__) @@ -1476,9 +1482,15 @@ def build_bm25_index(index, storage_dir: Path) -> None: seen_files.add(file_path) tokens = tokenize_filename(file_name) + node_metadata = {"file_name": file_name, "file_path": file_path} + # Revision metadata, so filename matches can be deduped per document + # and superseded revisions dropped from the matched-files list. + node_metadata.update( + {key: metadata[key] for key in ("doc_group", "superseded") if key in metadata} + ) filename_nodes.append(TextNode( text=" ".join(tokens), - metadata={"file_name": file_name, "file_path": file_path}, + metadata=node_metadata, id_=f"bm25_{file_path}" )) @@ -1743,9 +1755,10 @@ def build_index( new_count = 0 modified_count = 0 - # Phase 1: enumerate cheaply (stat only). Hashing happens in phase 2, - # and a file that fails either phase is recorded as unreadable so that - # its deletion is withheld below. + # Phase 1: enumerate cheaply (stat only), so revision grouping can + # discard files before they cost a hash - skipped files never get a + # state row, so no mtime fast path would save them. A file that fails + # either phase is recorded as unreadable so its deletion is withheld. ui.step_start("Scanning for changes") scan_count = 0 candidates: list[FileCandidate] = [] @@ -1763,15 +1776,46 @@ def build_index( ui.print(f"{ui.YELLOW}Interrupted during scan. No changes made.{ui.RESET}") return - # Phase 2: hash the candidates. + revision_policy = revisions.RevisionPolicy.from_config(_config) + grouping = revisions.partition( + [(c.path, c.mtime) for c in candidates], revision_policy + ) + for path, pattern in sorted(grouping.review_copies.items()): + logger.info("Skipping review copy %s (matched %r)", path, pattern) + if not revision_policy.index_superseded: + for path in sorted(grouping.superseded): + group = grouping.group_of.get(path, "") + logger.info( + "Skipping superseded revision %s (latest in group: %s)", + path, + grouping.latest_of_group.get(group, "?"), + ) + skipped_superseded = ( + 0 if revision_policy.index_superseded else len(grouping.superseded) + ) + + # Files whose chunks carry a superseded stamp from a previous run. + # Comparing the stamp against this scan's verdict lets flag flips + # (index_superseded toggled, a newer sibling arriving or vanishing, + # versioning disabled) propagate without a state-schema change. + currently_flagged: set[str] = set() + for doc in index.docstore.docs.values(): + metadata = doc.metadata or {} + if metadata.get("superseded") == "true" and metadata.get("file_path"): + currently_flagged.add(metadata["file_path"]) + + # Phase 2: hash only the keepers. + kept = set(grouping.keep) hashed_count = 0 for candidate in candidates: if abort_ctl.abort_requested: break + if candidate.path not in kept: + continue hashed_count += 1 if hashed_count % 100 == 0: ui.step_update( - f"Scanning for changes ({hashed_count:,}/{len(candidates):,} files)" + f"Scanning for changes ({hashed_count:,}/{len(kept):,} files)" ) try: file_info = data_source.file_info_for(candidate, tracked_files) @@ -1786,6 +1830,10 @@ def build_index( # without this line there is nothing to go on. if existing_state["hash"] != file_info.hash: reason = "content changed" + elif (file_info.path in grouping.superseded) != ( + file_info.path in currently_flagged + ): + reason = "revision status changed" else: reason = "" if reason: @@ -1838,6 +1886,10 @@ def build_index( parts.append(f"{len(deleted_files)} deleted") if withheld: parts.append(f"{len(withheld)} unseen (kept)") + if skipped_superseded: + parts.append(f"{skipped_superseded} superseded (skipped)") + if grouping.review_copies: + parts.append(f"{len(grouping.review_copies)} review copies (skipped)") ui.step_done(", ".join(parts) if parts else "no changes") # Process deletions @@ -1926,7 +1978,18 @@ def build_index( for file_path, outcome in file_docs.items(): file_doc_ids[file_path] = [] + group = grouping.group_of.get(file_path) + doc_id = grouping.doc_id_of.get(file_path) + superseded = file_path in grouping.superseded for doc in outcome.docs: + if group: + doc.metadata["doc_group"] = group + if doc_id: + doc.metadata["doc_id"] = doc_id + if superseded: + # Stored as a string: node metadata is flattened + # into the BM25 corpus and the docstore. + doc.metadata["superseded"] = "true" accumulated_docs.append(doc) file_doc_ids[file_path].append(doc.doc_id) @@ -2038,4 +2101,15 @@ def build_index( f"this scan did not see them, because the scan was incomplete. They " f"were NOT removed from the index. Re-run with --verbose to see why.{ui.RESET}" ) + if skipped_superseded or grouping.review_copies: + details = [] + if skipped_superseded: + details.append(f"{skipped_superseded} superseded revision(s)") + if grouping.review_copies: + details.append(f"{len(grouping.review_copies)} review copy file(s)") + ui.print( + f"{ui.YELLOW}{' and '.join(details)} were not indexed. Re-run with " + f"--verbose to list them; see indexing.versioning in config.yaml " + f"to adjust.{ui.RESET}" + ) diff --git a/src/chunksilo/revisions.py b/src/chunksilo/revisions.py new file mode 100644 index 0000000..f1d2d97 --- /dev/null +++ b/src/chunksilo/revisions.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +""" +Revision grouping for messy document version control. + +Documents in the indexed directories carry ad-hoc revision markers - a +revision-named directory (".../PB3/Spec.docx") or a token in the filename +("Spec PB3.docx", "Spec_v2.pdf") - with no consistent convention, plus copies +made to collect review comments. This module recognises files that are +revisions of one document, groups them, and picks the newest member of each +group by file modification time so the indexer can skip the rest. + +Revision labels are never ordered: tokens are only stripped from names to form +the group key. Which member is "latest" is decided by mtime alone. + +Pure string/path logic: no I/O, no config reads at import time. +""" +import re +import unicodedata +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import PurePath +from typing import Any, NamedTuple + +from .cfgload import DEFAULT_REVIEW_COPY_PATTERNS + +# Revision-token alternatives shared by filenames and directory names. Case +# rules are encoded per alternative, so nothing here compiles with IGNORECASE. +# +# Filename suffixes (stripped from the end of the stem, separator required): +_FILENAME_REV_ALTS = [ + # PA1, pb3, PC12 - preliminary marker. The digit is REQUIRED in filenames + # so initials like "Spec PM.docx" are not stripped. + r"[Pp][A-Za-z]\d{1,2}", + # A, B, C2 - single UPPERCASE letter only, so "Spec a.docx" and + # "Spec USA.docx" stay untouched. + r"[A-Z]\d{0,2}", + # v2, V1.0, v1_2 + r"[vV]\d+(?:[._]\d+){0,2}", + # rev2, Revision 3, ver_1, utg 2, utgåva B + r"(?:[Rr][Ee][Vv](?:ision)?|[Vv][Ee][Rr](?:sion)?|[Uu][Tt][Gg](?:[åa]va)?)" + r"[\s._\-]*(?:\d{1,3}|[A-Za-z]\d{0,2})", +] + +# Whole directory components. Anchored: "B" and "PB3" are revision dirs, +# "Backup", "B2B" and "2024" are not. Single letters are accepted in either +# case here (a dir named "b" is a revision dir; a file "Spec b.docx" is not). +_DIR_REV_ALTS = [ + # 1, 2, 3 - but not \d{4}: years are never revision dirs + r"\d{1,2}", + r"[A-Za-z]", + # PA, PB1 - the digit is optional for directories + r"[Pp][A-Za-z]\d{0,2}", + r"[vV]\d+(?:[._]\d+){0,2}", + r"(?:[Rr][Ee][Vv](?:ision)?|[Vv][Ee][Rr](?:sion)?|[Uu][Tt][Gg](?:[åa]va)?)" + r"[\s._\-]*(?:\d{1,3}|[A-Za-z]\d{0,2})?", +] + +# A trailing single letter after one of these words names a sibling document +# ("Appendix B"), not a revision, so the bare-letter alternative must not +# strip it. +_SINGLE_LETTER_GUARD = re.compile( + r"(?i:\b(?:appendix|annex|attachment|exhibit|schedule|part|section" + r"|chapter|figure|table|bilaga|del|kapitel|avsnitt|tabell|figur))[\s._\-]*$" +) + +_BARE_LETTER_TOKEN = re.compile(r"[A-Z]\d{0,2}$") + + +def _nfc(text: str) -> str: + # SMB/macOS mounts can deliver "å" decomposed (NFD), which would break + # every pattern containing it. + return unicodedata.normalize("NFC", text) + + +def _compile_alts(alts: list[str], extra: tuple[str, ...], template: str) -> re.Pattern: + parts = list(alts) + [f"(?i:{frag})" for frag in extra] + return re.compile(template.format(alts="|".join(parts))) + + +def _compile_user_patterns(patterns: Iterable[str], where: str) -> tuple[re.Pattern, ...]: + compiled = [] + for pat in patterns: + try: + compiled.append(re.compile(pat, re.IGNORECASE)) + except re.error as e: + raise ValueError(f"invalid regex in {where}: {pat!r} ({e})") from e + return tuple(compiled) + + +class Member(NamedTuple): + path: str + mtime: float + + +@dataclass +class GroupingResult: + """Outcome of partitioning a scanned file list.""" + + keep: list[str] = field(default_factory=list) # input order preserved + superseded: set[str] = field(default_factory=set) + review_copies: dict[str, str] = field(default_factory=dict) # path -> pattern + group_of: dict[str, str] = field(default_factory=dict) + doc_id_of: dict[str, str] = field(default_factory=dict) + latest_of_group: dict[str, str] = field(default_factory=dict) # key -> path + + +@dataclass(frozen=True) +class RevisionPolicy: + enabled: bool = True + index_superseded: bool = False + doc_id_patterns: tuple[re.Pattern, ...] = () + review_copy_patterns: tuple[re.Pattern, ...] = () + _filename_suffix: re.Pattern = None # type: ignore[assignment] + _dir_component: re.Pattern = None # type: ignore[assignment] + + @classmethod + def from_config(cls, config: dict[str, Any]) -> "RevisionPolicy": + cfg = (config.get("indexing") or {}).get("versioning") or {} + extra = tuple(cfg.get("extra_revision_tokens") or ()) + # Validate the extra fragments individually before folding them into + # the combined grammar, so a bad one is named in the error. + _compile_user_patterns(extra, "indexing.versioning.extra_revision_tokens") + review = cfg.get("review_copy_patterns") + if review is None: + review = DEFAULT_REVIEW_COPY_PATTERNS + return cls( + enabled=bool(cfg.get("enabled", True)), + index_superseded=bool(cfg.get("index_superseded", False)), + doc_id_patterns=_compile_user_patterns( + cfg.get("doc_id_patterns") or (), "indexing.versioning.doc_id_patterns" + ), + review_copy_patterns=_compile_user_patterns( + review, "indexing.versioning.review_copy_patterns" + ), + _filename_suffix=_compile_alts( + _FILENAME_REV_ALTS, extra, r"[\s._\-]+\(?({alts})\)?$" + ), + _dir_component=_compile_alts(_DIR_REV_ALTS, extra, r"^\(?(?:{alts})\)?$"), + ) + + def review_copy_match(self, path: str) -> str | None: + """Pattern that marks this file as a review copy, or None.""" + stem = _nfc(PurePath(path).stem) + for pattern in self.review_copy_patterns: + if pattern.search(stem): + return pattern.pattern + return None + + def extract_doc_id(self, path: str) -> str | None: + """Document ID from the full path, tried patterns in order.""" + normalized = _nfc(path) + for pattern in self.doc_id_patterns: + m = pattern.search(normalized) + if m: + return (m.group(1) if pattern.groups else m.group(0)).upper() + return None + + def is_revision_dir(self, component: str) -> bool: + return bool(self._dir_component.match(_nfc(component))) + + def strip_revision_suffixes(self, stem: str) -> str: + stem = _nfc(stem) + while True: + m = self._filename_suffix.search(stem) + if not m: + break + if _BARE_LETTER_TOKEN.fullmatch(m.group(1)) and _SINGLE_LETTER_GUARD.search( + stem[: m.start()] + ): + break + stem = stem[: m.start()] + # A stem that IS a revision token ("PB3.docx" inside a per-document + # folder) contributes nothing to identity. + if self._dir_component.match(stem): + return "" + return stem.rstrip(" ._-") + + def group_key(self, path: str) -> str: + doc_id = self.extract_doc_id(path) + if doc_id: + return f"id:{doc_id}" + pure = PurePath(path) + parent = "/".join( + part for part in pure.parent.parts if not self.is_revision_dir(part) + ) + stem = self.strip_revision_suffixes(pure.stem) + ext = pure.suffix.casefold() + if ext == ".doc": # the indexer converts .doc to .docx anyway + ext = ".docx" + return f"path:{_nfc(parent).casefold()}/{_nfc(stem).casefold()}{ext}" + + +def partition(files: Iterable[tuple[str, float]], policy: RevisionPolicy) -> GroupingResult: + """Split a scanned (path, mtime) list into keepers and skipped files. + + The latest member of each group is the one with the newest mtime (path + string breaks ties, purely for run-to-run determinism). With + ``policy.index_superseded`` the older members stay in ``keep`` but are + still listed in ``superseded`` so the indexer can stamp them. + """ + result = GroupingResult() + if not policy.enabled: + result.keep = [path for path, _ in files] + return result + + groups: dict[str, list[Member]] = {} + ordered: list[str] = [] + for path, mtime in files: + match = policy.review_copy_match(path) + if match is not None: + result.review_copies[path] = match + continue + ordered.append(path) + key = policy.group_key(path) + result.group_of[path] = key + doc_id = policy.extract_doc_id(path) + if doc_id: + result.doc_id_of[path] = doc_id + groups.setdefault(key, []).append(Member(path, mtime)) + + for key, members in groups.items(): + latest = max(members, key=lambda m: (m.mtime, m.path)) + result.latest_of_group[key] = latest.path + for member in members: + if member.path != latest.path: + result.superseded.add(member.path) + + if policy.index_superseded: + result.keep = ordered + else: + result.keep = [p for p in ordered if p not in result.superseded] + return result diff --git a/test/test_revision_indexing.py b/test/test_revision_indexing.py new file mode 100644 index 0000000..56725aa --- /dev/null +++ b/test/test_revision_indexing.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Integration tests for revision-aware indexing. + +Superseded revisions and review copies are filtered out of the scan, so the +existing deletion loop prunes previously-indexed ones, and an older revision +comes back automatically when the newest file disappears. +""" + +import contextlib +import copy +import json +import logging +import os +import sqlite3 +from unittest.mock import patch + +import pytest + +from chunksilo import index +from chunksilo.index import build_index, IndexConfig, DirectoryConfig + + +def _create_mock_embedding(): + from llama_index.core.embeddings import BaseEmbedding + + class MockEmbedding(BaseEmbedding): + def _get_vector(self, text: str): + vec = [0.0] * 384 + for word in text.lower().split(): + idx = sum(ord(c) for c in word) % 384 + vec[idx] += 1.0 + return vec + + def _get_query_embedding(self, query: str): + return self._get_vector(query) + + def _get_text_embedding(self, text: str): + return self._get_vector(text) + + async def _aget_query_embedding(self, query: str): + return self._get_vector(query) + + return MockEmbedding(model_name="mock-test") + + +class _LogSink(logging.Handler): + """Collects records from chunksilo.index (see test_incremental_ingest).""" + + def __init__(self): + super().__init__(level=logging.DEBUG) + self.messages: list[str] = [] + + def emit(self, record): + self.messages.append(record.getMessage()) + + @property + def text(self) -> str: + return "\n".join(self.messages) + + +@contextlib.contextmanager +def capture_index_log(): + sink = _LogSink() + log = logging.getLogger("chunksilo.index") + previous = log.level + log.setLevel(logging.DEBUG) + log.addHandler(sink) + try: + yield sink + finally: + log.removeHandler(sink) + log.setLevel(previous) + + +@contextlib.contextmanager +def versioning_config(**overrides): + """Run with indexing.versioning options overridden in index._config.""" + orig = index._config + cfg = copy.deepcopy(orig) + cfg.setdefault("indexing", {}).setdefault("versioning", {}).update(overrides) + index._config = cfg + try: + yield + finally: + index._config = orig + + +@pytest.fixture +def test_env(tmp_path): + data_dir = tmp_path / "data" + data_dir.mkdir() + storage_dir = tmp_path / "storage" + db_path = storage_dir / "ingestion_state.db" + + test_index_config = IndexConfig( + directories=[DirectoryConfig(path=data_dir)], + chunk_size=512, + chunk_overlap=100, + ) + + orig_storage_dir = index.STORAGE_DIR + orig_db_path = index.STATE_DB_PATH + + index.STORAGE_DIR = storage_dir + index.STATE_DB_PATH = db_path + + mock_embed = _create_mock_embedding() + + with patch("chunksilo.index.load_index_config", return_value=test_index_config), \ + patch("chunksilo.index._create_fastembed_embedding", return_value=mock_embed), \ + patch("chunksilo.index.ensure_embedding_model_cached"), \ + patch("chunksilo.index.ensure_rerank_model_cached"): + yield data_dir, storage_dir, db_path + + index.STORAGE_DIR = orig_storage_dir + index.STATE_DB_PATH = orig_db_path + + +def create_file(data_dir, name, content, mtime=None): + path = data_dir / name + path.write_text(content) + if mtime is not None: + os.utime(path, (mtime, mtime)) + return path + + +def tracked_names(db_path): + if not db_path.exists(): + return set() + with sqlite3.connect(db_path) as conn: + return { + os.path.basename(row[0]) + for row in conn.execute("SELECT path FROM files") + } + + +def docstore_metadata(storage_dir): + """{file_name: merged metadata} straight from the persisted docstore.""" + docstore = json.loads((storage_dir / "docstore.json").read_text()) + result = {} + for entry in docstore.get("docstore/data", {}).values(): + metadata = entry.get("__data__", {}).get("metadata", {}) + name = metadata.get("file_name") + if name: + result.setdefault(name, {}).update(metadata) + return result + + +T0, T1, T2, T3 = 1_600_000_000.0, 1_600_100_000.0, 1_600_200_000.0, 1_600_300_000.0 + + +def test_fresh_build_keeps_only_newest(test_env): + data_dir, storage_dir, db_path = test_env + create_file(data_dir, "Spec PB1.txt", "spec preliminary one", mtime=T0) + create_file(data_dir, "Spec PB2.txt", "spec preliminary two", mtime=T1) + create_file(data_dir, "Spec B.txt", "spec released", mtime=T2) + create_file(data_dir, "Spec B (review comments JD).txt", "commented", mtime=T3) + create_file(data_dir, "Other.txt", "unrelated document", mtime=T0) + + build_index() + + assert tracked_names(db_path) == {"Spec B.txt", "Other.txt"} + metadata = docstore_metadata(storage_dir) + assert set(metadata) == {"Spec B.txt", "Other.txt"} + assert metadata["Spec B.txt"]["doc_group"].startswith("path:") + assert "superseded" not in metadata["Spec B.txt"] + + +def test_revision_dirs_group_and_newest_wins(test_env): + data_dir, storage_dir, db_path = test_env + (data_dir / "Spec" / "PB1").mkdir(parents=True) + (data_dir / "Spec" / "B").mkdir(parents=True) + create_file(data_dir, "Spec/PB1/Spec.txt", "preliminary", mtime=T0) + create_file(data_dir, "Spec/B/Spec.txt", "released", mtime=T1) + + build_index() + + with sqlite3.connect(db_path) as conn: + paths = [row[0] for row in conn.execute("SELECT path FROM files")] + assert len(paths) == 1 and paths[0].endswith("B/Spec.txt") + + +def test_new_revision_prunes_old(test_env): + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "Spec PB1.txt", "preliminary", mtime=T0) + build_index() + assert tracked_names(db_path) == {"Spec PB1.txt"} + + create_file(data_dir, "Spec B.txt", "released", mtime=T1) + build_index() + assert tracked_names(db_path) == {"Spec B.txt"} + + +def test_newest_deleted_resurrects_older(test_env): + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "Spec PB1.txt", "preliminary", mtime=T0) + create_file(data_dir, "Spec B.txt", "released", mtime=T1) + build_index() + assert tracked_names(db_path) == {"Spec B.txt"} + + (data_dir / "Spec B.txt").unlink() + build_index() + assert tracked_names(db_path) == {"Spec PB1.txt"} + + +def test_first_versioned_run_prunes_existing(test_env): + """Upgrade path: files indexed before versioning existed get pruned.""" + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "Spec PB1.txt", "preliminary", mtime=T0) + create_file(data_dir, "Spec B.txt", "released", mtime=T1) + with versioning_config(enabled=False): + build_index() + assert tracked_names(db_path) == {"Spec PB1.txt", "Spec B.txt"} + + build_index() + assert tracked_names(db_path) == {"Spec B.txt"} + + +def test_disabled_indexes_everything(test_env): + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "Spec PB1.txt", "preliminary", mtime=T0) + create_file(data_dir, "Spec B.txt", "released", mtime=T1) + create_file(data_dir, "Copy of Spec.txt", "copied", mtime=T2) + with versioning_config(enabled=False): + build_index() + assert tracked_names(db_path) == {"Spec PB1.txt", "Spec B.txt", "Copy of Spec.txt"} + + +def test_index_superseded_stamps_and_tracks_flips(test_env): + data_dir, storage_dir, db_path = test_env + create_file(data_dir, "Spec PB1.txt", "preliminary", mtime=T0) + create_file(data_dir, "Spec B.txt", "released", mtime=T1) + with versioning_config(index_superseded=True): + build_index() + assert tracked_names(db_path) == {"Spec PB1.txt", "Spec B.txt"} + metadata = docstore_metadata(storage_dir) + assert metadata["Spec PB1.txt"]["superseded"] == "true" + assert "superseded" not in metadata["Spec B.txt"] + assert metadata["Spec PB1.txt"]["doc_group"] == metadata["Spec B.txt"]["doc_group"] + + # A newer revision arrives: the previous latest is unchanged on disk but + # its stamp must flip, so it is reprocessed. + create_file(data_dir, "Spec C.txt", "newer release", mtime=T2) + with versioning_config(index_superseded=True): + with capture_index_log() as log: + build_index() + assert "revision status changed" in log.text + metadata = docstore_metadata(storage_dir) + assert metadata["Spec B.txt"]["superseded"] == "true" + assert "superseded" not in metadata["Spec C.txt"] + + # Turning the option back off prunes the old revisions. + build_index() + assert tracked_names(db_path) == {"Spec C.txt"} + + +def test_stale_stamp_cleaned_when_versioning_disabled(test_env): + """Disabling versioning entirely also clears leftover superseded stamps.""" + data_dir, storage_dir, db_path = test_env + create_file(data_dir, "Spec PB1.txt", "preliminary", mtime=T0) + create_file(data_dir, "Spec B.txt", "released", mtime=T1) + with versioning_config(index_superseded=True): + build_index() + assert docstore_metadata(storage_dir)["Spec PB1.txt"]["superseded"] == "true" + + with versioning_config(enabled=False): + with capture_index_log() as log: + build_index() + assert "revision status changed" in log.text + assert "superseded" not in docstore_metadata(storage_dir)["Spec PB1.txt"] + + +def test_unchanged_keeper_not_reprocessed(test_env): + data_dir, _storage_dir, _db_path = test_env + create_file(data_dir, "Spec PB1.txt", "preliminary", mtime=T0) + create_file(data_dir, "Spec B.txt", "released", mtime=T1) + build_index() + + with capture_index_log() as log: + build_index() + assert "Reprocessing" not in log.text + assert "Indexing new file" not in log.text + + +def test_skip_reasons_logged(test_env): + data_dir, _storage_dir, _db_path = test_env + create_file(data_dir, "Spec PB1.txt", "preliminary", mtime=T0) + create_file(data_dir, "Spec B.txt", "released", mtime=T1) + create_file(data_dir, "Copy of Spec.txt", "copied", mtime=T2) + + with capture_index_log() as log: + build_index() + assert "Skipping superseded revision" in log.text + assert "Skipping review copy" in log.text diff --git a/test/test_revisions.py b/test/test_revisions.py new file mode 100644 index 0000000..8775fc4 --- /dev/null +++ b/test/test_revisions.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Unit tests for revision grouping (revisions.py). + +Pure string/path logic: no filesystem, no config files. +""" + +import unicodedata + +import pytest + +from chunksilo.revisions import RevisionPolicy, partition + + +@pytest.fixture +def policy(): + return RevisionPolicy.from_config({}) + + +# ============================================================================= +# Filename suffix stripping +# ============================================================================= + + +class TestStripRevisionSuffixes: + @pytest.mark.parametrize( + "stem,expected", + [ + ("Spec PB3", "Spec"), + ("Spec pb3", "Spec"), + ("Spec_v2", "Spec"), + ("Spec V1.0", "Spec"), + ("Spec A", "Spec"), + ("Spec B2", "Spec"), + ("Report rev 3", "Report"), + ("Report Revision 3", "Report"), + ("Plan utgåva 2", "Plan"), + ("Plan utg 2", "Plan"), + ("Spec (PB3)", "Spec"), + ("Spec v2 PB1", "Spec"), # iterative + ("Spec-B", "Spec"), + ], + ) + def test_stripped(self, policy, stem, expected): + assert policy.strip_revision_suffixes(stem) == expected + + @pytest.mark.parametrize( + "stem", + [ + "Vitamin", # no separator, nothing to strip + "Spec USA", # multi-letter uppercase is not a revision + "Report 2024", # bare digits never strip from filenames + "Chapter 3", + "Spec final", + "Spec a", # lowercase single letter is a word, not a revision + "Spec PM", # P-form without digit = initials + "CV", # no separator + ], + ) + def test_untouched(self, policy, stem): + assert policy.strip_revision_suffixes(stem) == stem + + def test_nfd_input_normalized(self, policy): + decomposed = unicodedata.normalize("NFD", "Plan utgåva 2") + assert policy.strip_revision_suffixes(decomposed) == "Plan" + + @pytest.mark.parametrize( + "stem", + ["Appendix B", "Annex A", "Bilaga C", "Part B", "Chapter A", "Tabell B"], + ) + def test_single_letter_guard(self, policy, stem): + """A letter naming a sibling document is not a revision.""" + assert policy.strip_revision_suffixes(stem) == stem + + def test_whole_stem_revision_token(self, policy): + """A stem that IS a revision token carries no identity.""" + assert policy.strip_revision_suffixes("PB3") == "" + assert policy.strip_revision_suffixes("v2") == "" + assert policy.strip_revision_suffixes("B") == "" + + +# ============================================================================= +# Directory components +# ============================================================================= + + +class TestIsRevisionDir: + @pytest.mark.parametrize( + "component", ["PB3", "PA", "B", "b", "2", "12", "v1.0", "V2", "Rev", "Revision 2", "utgåva 3", "(PB3)"] + ) + def test_revision_dirs(self, policy, component): + assert policy.is_revision_dir(component) + + @pytest.mark.parametrize( + "component", ["Backup", "B2B", "2024", "123", "Specs", "PBX123", "v2-final"] + ) + def test_normal_dirs(self, policy, component): + assert not policy.is_revision_dir(component) + + +# ============================================================================= +# Review-copy detection +# ============================================================================= + + +class TestReviewCopyDetection: + @pytest.mark.parametrize( + "name", + [ + "/x/Spec B (review comments JD).docx", + "/x/Copy of Spec.docx", + "/x/Spec - Copy (2).docx", + "/x/Kopia av Spec.docx", + "/x/Spec - kopia.docx", + "/x/Spec B granskning.docx", + "/x/Spec kommentarer.docx", + ], + ) + def test_detected(self, policy, name): + assert policy.review_copy_match(name) is not None + + def test_word_boundary(self, policy): + assert policy.review_copy_match("/x/copyright.docx") is None + assert policy.review_copy_match("/x/Spec.docx") is None + + def test_known_over_match_trade_off(self, policy): + # Documented default behavior: a document legitimately named after the + # review activity matches too. The skip is logged per file and the + # patterns are configurable. + assert policy.review_copy_match("/x/Literature Review.docx") is not None + + def test_directory_names_ignored(self, policy): + assert policy.review_copy_match("/reviews/Spec.docx") is None + + +# ============================================================================= +# Doc-ID extraction and group keys +# ============================================================================= + + +class TestGroupKey: + def test_doc_id_pattern_wins_over_path(self): + policy = RevisionPolicy.from_config( + {"indexing": {"versioning": {"doc_id_patterns": [r"\b(DOC-\d{5})\b"]}}} + ) + a = policy.group_key("/share/proj1/PB1/DOC-12345 Spec.docx") + b = policy.group_key("/share/archive/DOC-12345 Spec B.docx") + assert a == b == "id:DOC-12345" + + def test_doc_id_whole_match_when_no_group(self): + policy = RevisionPolicy.from_config( + {"indexing": {"versioning": {"doc_id_patterns": [r"DOC\d{5}"]}}} + ) + assert policy.extract_doc_id("/x/notes.pdf") is None + assert policy.extract_doc_id("/x/DOC12345 spec.pdf") == "DOC12345" + # Matching is case-insensitive and the ID is upper-cased, consistent + # with the case-folded path keys. + assert policy.extract_doc_id("/x/doc12345 spec.pdf") == "DOC12345" + + def test_bad_doc_id_pattern_raises(self): + with pytest.raises(ValueError, match=r"doc_id_patterns.*\("): + RevisionPolicy.from_config( + {"indexing": {"versioning": {"doc_id_patterns": ["("]}}} + ) + + def test_revision_dirs_stripped_from_parent(self, policy): + a = policy.group_key("/share/proj/Spec/PB1/Spec.docx") + b = policy.group_key("/share/proj/Spec/PB2/Spec.docx") + c = policy.group_key("/share/proj/Spec/B/Spec.docx") + assert a == b == c + + def test_filename_revisions_group(self, policy): + assert policy.group_key("/share/Spec PB3.docx") == policy.group_key( + "/share/Spec B.docx" + ) + + def test_different_directories_do_not_group(self, policy): + assert policy.group_key("/share/proj1/Spec.docx") != policy.group_key( + "/share/proj2/Spec.docx" + ) + + def test_extension_is_part_of_key(self, policy): + assert policy.group_key("/share/Spec B.pdf") != policy.group_key( + "/share/Spec B.docx" + ) + + def test_doc_and_docx_group(self, policy): + assert policy.group_key("/share/Spec A.doc") == policy.group_key( + "/share/Spec B.docx" + ) + + def test_case_insensitive_keys(self, policy): + assert policy.group_key("/share/SPEC B.DOCX") == policy.group_key( + "/share/spec.docx" + ) + + def test_whole_stem_revision_groups_by_folder(self, policy): + a = policy.group_key("/share/proj/Spec/PB1.docx") + b = policy.group_key("/share/proj/Spec/B.docx") + assert a == b + + def test_extra_revision_tokens(self): + policy = RevisionPolicy.from_config( + {"indexing": {"versioning": {"extra_revision_tokens": [r"draft\d+"]}}} + ) + assert policy.strip_revision_suffixes("Spec draft3") == "Spec" + assert policy.is_revision_dir("Draft2") + + +# ============================================================================= +# Partitioning +# ============================================================================= + + +class TestPartition: + def test_latest_by_mtime_wins(self, policy): + files = [ + ("/s/Spec PB1.docx", 100.0), + ("/s/Spec PB2.docx", 200.0), + ("/s/Spec B.docx", 300.0), + ] + result = partition(files, policy) + assert result.keep == ["/s/Spec B.docx"] + assert result.superseded == {"/s/Spec PB1.docx", "/s/Spec PB2.docx"} + key = result.group_of["/s/Spec B.docx"] + assert result.latest_of_group[key] == "/s/Spec B.docx" + + def test_mtime_beats_label(self, policy): + # The decided rule: mtime alone picks the latest, labels never order. + files = [("/s/Spec B.docx", 100.0), ("/s/Spec PB1.docx", 200.0)] + result = partition(files, policy) + assert result.keep == ["/s/Spec PB1.docx"] + + def test_tie_break_deterministic(self, policy): + files = [("/s/Spec A.docx", 100.0), ("/s/Spec B.docx", 100.0)] + assert partition(files, policy).keep == partition(list(reversed(files)), policy).keep + + def test_ungrouped_files_all_kept_in_order(self, policy): + files = [("/s/One.docx", 1.0), ("/t/Two.pdf", 2.0), ("/u/Three.md", 3.0)] + result = partition(files, policy) + assert result.keep == [p for p, _ in files] + assert not result.superseded + + def test_review_copies_always_skipped(self, policy): + files = [ + ("/s/Spec B.docx", 100.0), + ("/s/Spec B (review comments JD).docx", 999.0), + ] + result = partition(files, policy) + assert result.keep == ["/s/Spec B.docx"] + assert "/s/Spec B (review comments JD).docx" in result.review_copies + + def test_disabled_keeps_everything(self): + policy = RevisionPolicy.from_config( + {"indexing": {"versioning": {"enabled": False}}} + ) + files = [("/s/Spec PB1.docx", 1.0), ("/s/Spec B.docx", 2.0), ("/s/Copy of X.docx", 3.0)] + result = partition(files, policy) + assert result.keep == [p for p, _ in files] + assert not result.superseded + assert not result.review_copies + + def test_index_superseded_keeps_all_but_marks(self): + policy = RevisionPolicy.from_config( + {"indexing": {"versioning": {"index_superseded": True}}} + ) + files = [("/s/Spec PB1.docx", 1.0), ("/s/Spec B.docx", 2.0)] + result = partition(files, policy) + assert result.keep == ["/s/Spec PB1.docx", "/s/Spec B.docx"] + assert result.superseded == {"/s/Spec PB1.docx"} + + def test_doc_id_of_populated(self): + policy = RevisionPolicy.from_config( + {"indexing": {"versioning": {"doc_id_patterns": [r"(\d{4}-\d{2})"]}}} + ) + files = [("/s/1234-01 Spec.docx", 1.0)] + result = partition(files, policy) + assert result.doc_id_of["/s/1234-01 Spec.docx"] == "1234-01" From 2d32dba6bb9bba23bde740af3462eac6eea25375 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:14:10 +0200 Subject: [PATCH 06/18] feat: add --check-files to dry-run the indexing file filters chunksilo --check-files [PATH] runs the indexer's exact phase-1 scan (availability, directory pruning, include/exclude globs, revision grouping, review copies) and reports a per-file verdict with the deciding reason - read-only: no models, no hashing, no state-DB writes. With PATH it explains one file, including its revision group. Exit codes make it scriptable; --json emits the full report. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- src/chunksilo/cli.py | 32 ++- src/chunksilo/filecheck.py | 442 ++++++++++++++++++++++++++++++ test/test_check_files.py | 538 +++++++++++++++++++++++++++++++++++++ 3 files changed, 1011 insertions(+), 1 deletion(-) create mode 100644 src/chunksilo/filecheck.py create mode 100644 test/test_check_files.py diff --git a/src/chunksilo/cli.py b/src/chunksilo/cli.py index acdfd63..75bd0bd 100644 --- a/src/chunksilo/cli.py +++ b/src/chunksilo/cli.py @@ -7,6 +7,7 @@ chunksilo "query text" [--date-from YYYY-MM-DD] [--date-to YYYY-MM-DD] [--config PATH] [--json] chunksilo --build-index [--config PATH] chunksilo --download-models [--config PATH] + chunksilo --check-files [PATH] [--config PATH] [--json] """ import argparse import json @@ -16,6 +17,23 @@ from pathlib import Path +def _check_files(target: str | None, config_path: Path | None, as_json: bool) -> int: + """Dry-run the indexing file filters and print per-file verdicts.""" + from . import filecheck + from . import index as index_module + from .cfgload import load_config + + config = load_config(config_path) + index_module._config = config + + report = filecheck.run_check(config, target=target) + if as_json: + print(json.dumps(filecheck.to_json(report), indent=2)) + else: + filecheck.render_text(report) + return filecheck.exit_code(report) + + def main(): """Entry point for the `chunksilo` command.""" os.environ["TOKENIZERS_PARALLELISM"] = "false" @@ -48,6 +66,11 @@ def main(): help="Print all default configuration values as YAML, then exit") parser.add_argument("--list-files", action="store_true", help="List all indexed file paths, then exit") + parser.add_argument("--check-files", nargs="?", const=True, default=None, + metavar="PATH", + help="Dry-run the indexing file filters and report a verdict " + "per file, without building the index. With PATH, " + "explain that one file, then exit") args = parser.parse_args() @@ -86,6 +109,12 @@ def main(): print(p) return + if args.check_files is not None: + if args.query: + parser.error("--check-files cannot be combined with a search query") + target = None if args.check_files is True else args.check_files + sys.exit(_check_files(target, config_path, args.json)) + if args.build_index or args.download_models: # Suppress 3rd-party native output when IndexingUI owns the terminal if not args.verbose: @@ -102,7 +131,8 @@ def main(): return if not args.query: - parser.error("query is required (or use --build-index / --list-files / --download-models)") + parser.error("query is required (or use --build-index / --list-files / " + "--check-files / --download-models)") from .search import run_search diff --git a/src/chunksilo/filecheck.py b/src/chunksilo/filecheck.py new file mode 100644 index 0000000..e0763ba --- /dev/null +++ b/src/chunksilo/filecheck.py @@ -0,0 +1,442 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Dry-run of the indexing scan: same filters, no index, no writes. + +Runs the exact phase-1 scan the indexer uses (directory availability, +timeout-guarded walk, directory pruning, glob matching, stat, cross-source +dedup, revision partitioning) and reports a per-file verdict with the +deciding reason. Never loads models, hashes files, or writes anywhere - +the state database is only ever opened read-only. +""" +import sqlite3 +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, TextIO + +from . import revisions + +# Verdict statuses, in the order they appear in reports. +STATUSES = ( + "index", + "superseded", + "review_copy", + "excluded", + "not_included", + "unreadable", + "duplicate", +) + +_STATUS_LABELS = { + "index": "would index", + "superseded": "superseded", + "review_copy": "review copy", + "excluded": "excluded", + "not_included": "not included", + "unreadable": "unreadable", + "duplicate": "duplicate", +} + + +@dataclass +class FileVerdict: + path: str + status: str # one of STATUSES + reason: str + pattern: str | None = None # glob or review-copy regex that decided + group: str | None = None # revision group key + latest_in_group: str | None = None + indexed: bool | None = None # in the state DB (None = DB absent or n/a) + + +@dataclass +class PrunedDir: + path: str + pattern: str + + +@dataclass +class DirectoryReport: + path: str # absolute + enabled: bool + available: bool + unavailable_reason: str | None = None + recursive: bool = True + scan_complete: bool = True + scan_incomplete_reason: str | None = None + pruned_dirs: list[PrunedDir] = field(default_factory=list) + files: list[FileVerdict] = field(default_factory=list) + counts: dict[str, int] = field(default_factory=dict) + + +@dataclass +class CheckReport: + directories: list[DirectoryReport] = field(default_factory=list) + totals: dict[str, int] = field(default_factory=dict) + state_db: str = "" + state_db_present: bool = False + target: FileVerdict | None = None # single-path mode + error: str | None = None # config-level failure + + +def run_check(config: dict[str, Any], target: str | None = None) -> CheckReport: + """Scan all configured directories with the indexer's filters, no writes. + + The caller must have activated ``config`` process-wide (load_config plus + index._config) so cfgload.get() inside the walk answers from the same + file; the CLI handler does this. + """ + from . import index as index_module + + state_db = Path(config["storage"]["storage_dir"]) / "ingestion_state.db" + report = CheckReport(state_db=str(state_db)) + + try: + index_config = index_module._parse_index_config(config.get("indexing") or {}) + except ValueError as exc: + report.error = str(exc) + return report + + reports_by_entry: dict[int, DirectoryReport] = {} + for dc in index_config.directories: + dir_report = DirectoryReport( + path=str(dc.path.absolute()), + enabled=dc.enabled, + available=False, + recursive=dc.recursive, + ) + if not dc.enabled: + dir_report.unavailable_reason = "disabled in config (enabled: false)" + reports_by_entry[id(dc)] = dir_report + report.directories.append(dir_report) + + data_source = index_module.MultiDirectoryDataSource(index_config) + + for dc in data_source.unavailable_dirs: + dir_report = reports_by_entry[id(dc)] + dir_report.unavailable_reason = data_source.unscanned_roots().get( + str(dc.path.absolute()), "directory unavailable" + ) + + # Phase 1 scan, source by source, with one shared seen-set replicating + # MultiDirectoryDataSource.iter_candidates dedup (first source wins). + seen: dict[str, str] = {} # abs path -> source dir that saw it first + candidates: list[Any] = [] # FileCandidate, in scan order + verdict_report: dict[str, DirectoryReport] = {} # candidate path -> report + source_of: dict[str, Any] = {} # candidate path -> LocalFileSystemSource + + for source in data_source.sources: + dir_report = reports_by_entry[id(source.config)] + dir_report.available = True + + events: list[Any] = [] + for candidate in source.iter_candidates(on_event=events.append): + if candidate.path in seen: + dir_report.files.append(FileVerdict( + path=candidate.path, + status="duplicate", + reason=f"already scanned via {seen[candidate.path]}", + )) + continue + seen[candidate.path] = candidate.source_dir + candidates.append(candidate) + verdict_report[candidate.path] = dir_report + source_of[candidate.path] = source + + dir_report.scan_complete = source.scan_status.complete + if not source.scan_status.complete: + dir_report.scan_incomplete_reason = source.scan_status.reason + + for event in events: + if event.kind == "pruned_dir": + dir_report.pruned_dirs.append(PrunedDir(event.path, event.pattern)) + elif event.kind == "excluded_file": + if event.pattern is not None: + dir_report.files.append(FileVerdict( + path=event.path, + status="excluded", + reason=f"excluded by pattern {event.pattern!r}", + pattern=event.pattern, + )) + else: + dir_report.files.append(FileVerdict( + path=event.path, + status="not_included", + reason="no include pattern matched", + )) + elif event.kind == "unreadable_file": + dir_report.files.append(FileVerdict( + path=event.path, + status="unreadable", + reason="could not stat (error or timeout); " + "would be skipped and its deletion withheld", + )) + + # Revision partitioning across ALL sources at once, exactly like + # build_index - doc-ID groups can span directories. + policy = revisions.RevisionPolicy.from_config(config) + grouping = revisions.partition( + [(c.path, c.mtime) for c in candidates], policy + ) + + tracked, report.state_db_present = _read_state_db(state_db) + + for candidate in candidates: + path = candidate.path + dir_report = verdict_report[path] + group = grouping.group_of.get(path) + latest = grouping.latest_of_group.get(group) if group else None + + if path in grouping.review_copies: + pattern = grouping.review_copies[path] + dir_report.files.append(FileVerdict( + path=path, + status="review_copy", + reason=f"review copy (filename matched {pattern!r}); never indexed", + pattern=pattern, + )) + continue + + if path in grouping.superseded and not policy.index_superseded: + dir_report.files.append(FileVerdict( + path=path, + status="superseded", + reason=f"older revision; latest in group is {latest}", + group=group, + latest_in_group=latest, + )) + continue + + # Kept: re-derive the include pattern (pure string work, no I/O). + _, pattern = source_of[path]._match_decision(Path(path)) + if path in grouping.superseded: + reason = ( + "older revision, indexed and stamped superseded " + f"(index_superseded: true); latest in group is {latest}" + ) + elif pattern is not None: + reason = f"matched include pattern {pattern!r}" + else: + reason = "no include patterns configured; everything passes" + dir_report.files.append(FileVerdict( + path=path, + status="index", + reason=reason, + pattern=pattern, + group=group, + latest_in_group=latest if latest != path else None, + indexed=(path in tracked) if report.state_db_present else None, + )) + + for dir_report in report.directories: + dir_report.files.sort(key=lambda v: v.path) + dir_report.pruned_dirs.sort(key=lambda p: p.path) + dir_report.counts = {status: 0 for status in STATUSES} + for verdict in dir_report.files: + dir_report.counts[verdict.status] += 1 + dir_report.counts["pruned_dirs"] = len(dir_report.pruned_dirs) + + report.totals = {status: 0 for status in STATUSES} + report.totals["pruned_dirs"] = 0 + for dir_report in report.directories: + for key, count in dir_report.counts.items(): + report.totals[key] += count + + if target is not None: + report.target = _resolve_target(target, report, data_source) + + return report + + +def _read_state_db(state_db: Path) -> tuple[set[str], bool]: + """Indexed paths from the state DB, strictly read-only; never creates it.""" + if not state_db.exists(): + return set(), False + try: + with sqlite3.connect(f"file:{state_db}?mode=ro", uri=True) as conn: + return {row[0] for row in conn.execute("SELECT path FROM files")}, True + except sqlite3.OperationalError: + return set(), True + + +def _under(path: str, root: str) -> bool: + return path == root or path.startswith(root.rstrip("/") + "/") + + +def _resolve_target(target, report, data_source): + """Verdict for one path, using the full scan that already ran.""" + path = str(Path(target).expanduser().absolute()) + + for dir_report in report.directories: + for verdict in dir_report.files: + if verdict.path == path: + return verdict + for pruned in dir_report.pruned_dirs: + if _under(path, pruned.path): + return FileVerdict( + path=path, + status="excluded", + reason=f"ancestor directory {pruned.path} pruned " + f"by pattern {pruned.pattern!r}", + pattern=pruned.pattern, + ) + + for dir_report in report.directories: + if not _under(path, dir_report.path): + continue + if not dir_report.enabled: + return FileVerdict( + path=path, + status="excluded", + reason=f"directory {dir_report.path} is disabled (enabled: false)", + ) + if not dir_report.available: + return FileVerdict( + path=path, + status="unreadable", + reason=f"directory {dir_report.path} is unavailable " + f"({dir_report.unavailable_reason})", + ) + if not dir_report.recursive and str(Path(path).parent) != dir_report.path: + return FileVerdict( + path=path, + status="excluded", + reason=f"directory {dir_report.path} is configured " + "recursive: false; only top-level files are scanned", + ) + # Under an available directory but the scan produced no verdict: + # explain from the matcher directly. + for source in data_source.sources: + if str(source.base_dir.absolute()) != dir_report.path: + continue + matched, pattern = source._match_decision(Path(path)) + exists = Path(path).is_file() + if not matched and pattern is not None: + reason = f"excluded by pattern {pattern!r}" + status = "excluded" + elif not matched: + reason = "no include pattern matched" + status = "not_included" + else: + status = "excluded" + reason = "matches the patterns but the scan did not see it" + if not dir_report.scan_complete: + reason += f" (scan incomplete: {dir_report.scan_incomplete_reason})" + if not exists: + reason += "; file does not exist" + return FileVerdict(path=path, status=status, reason=reason, + pattern=pattern if not matched else None) + + return FileVerdict( + path=path, + status="excluded", + reason="not under any configured indexing directory", + ) + + +def exit_code(report: CheckReport) -> int: + """0 = clean (or target would be indexed); 1 = problem or target skipped.""" + if report.error: + return 1 + if report.target is not None: + return 0 if report.target.status == "index" else 1 + for dir_report in report.directories: + if dir_report.enabled and ( + not dir_report.available or not dir_report.scan_complete + ): + return 1 + return 0 + + +def to_json(report: CheckReport) -> dict[str, Any]: + from dataclasses import asdict + + data = asdict(report) + # asdict turns PrunedDir into {"path", "pattern"} dicts already. + return data + + +def render_text(report: CheckReport, out: TextIO | None = None) -> None: + if out is None: + out = sys.stdout + if report.error: + print(f"Configuration error: {report.error}", file=out) + return + + if report.target is not None: + _render_target(report, out) + return + + db_note = "present" if report.state_db_present else "not created yet" + print(f"State DB: {report.state_db} ({db_note})", file=out) + + for dir_report in report.directories: + print(file=out) + print(f"{dir_report.path}", file=out) + if not dir_report.enabled: + print(" disabled in config (enabled: false) - not scanned", file=out) + continue + if not dir_report.available: + print(f" UNAVAILABLE: {dir_report.unavailable_reason}", file=out) + continue + mode = "recursive" if dir_report.recursive else "top-level files only" + print(f" scanned ({mode})", file=out) + if not dir_report.scan_complete: + print(f" WARNING: scan incomplete - {dir_report.scan_incomplete_reason}; " + "files under unvisited subtrees are missing from this report", + file=out) + for pruned in dir_report.pruned_dirs: + print(f" pruned dir {pruned.path} [by {pruned.pattern!r}]", file=out) + for verdict in dir_report.files: + label = _STATUS_LABELS[verdict.status].ljust(13) + suffix = "" + if verdict.status == "index": + if verdict.indexed is True: + suffix = " (indexed)" + elif verdict.indexed is False: + suffix = " (not indexed yet)" + print(f" {label} {verdict.path} [{verdict.reason}]{suffix}", file=out) + + print(file=out) + print(f"Summary: {_summary_line(report.totals)}", file=out) + + +def _render_target(report: CheckReport, out: TextIO) -> None: + verdict = report.target + print(f"Path: {verdict.path}", file=out) + label = _STATUS_LABELS[verdict.status] + print(f"Verdict: {label} - {verdict.reason}", file=out) + if verdict.status == "index": + if verdict.indexed is True: + print("Index: already present in the state DB", file=out) + elif verdict.indexed is False: + print("Index: not indexed yet - run chunksilo --build-index", file=out) + if verdict.group: + members = [ + other + for dir_report in report.directories + for other in dir_report.files + if other.group == verdict.group and other.path != verdict.path + ] + if members: + print(f"Revision group: {verdict.group}", file=out) + for member in members: + print(f" {_STATUS_LABELS[member.status].ljust(13)} {member.path}", + file=out) + + +def _summary_line(totals: dict[str, int]) -> str: + parts = [] + names = { + "index": "would index", + "superseded": "superseded", + "review_copy": "review copies", + "excluded": "excluded", + "not_included": "not matching any include", + "unreadable": "unreadable", + "duplicate": "duplicates", + "pruned_dirs": "directories pruned", + } + for key, name in names.items(): + if totals.get(key): + parts.append(f"{totals[key]} {name}") + return ", ".join(parts) if parts else "nothing found" diff --git a/test/test_check_files.py b/test/test_check_files.py new file mode 100644 index 0000000..8cf0a9d --- /dev/null +++ b/test/test_check_files.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +"""Tests for the --check-files dry-run diagnostics. + +Covers the reason-returning matchers, the ScanEvent callback on +iter_candidates, filecheck.run_check verdicts, and the CLI wiring. +The check must be strictly read-only: it must never create the state DB +or write anywhere. +""" +import json +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from chunksilo import filecheck +from chunksilo.index import ( + DirectoryConfig, + FileInfo, + IngestionState, + LocalFileSystemSource, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_source(tmp_path, **overrides): + """Create a LocalFileSystemSource pointing at tmp_path.""" + cfg = DirectoryConfig(path=tmp_path, **overrides) + return LocalFileSystemSource(cfg) + + +def _config_for(tmp_path, dirs, **indexing_extra): + """Minimal config dict for run_check.""" + indexing = {"directories": dirs} + indexing.update(indexing_extra) + return { + "storage": {"storage_dir": str(tmp_path / "storage")}, + "indexing": indexing, + } + + +def _verdict_by_name(report, name): + """The FileVerdict whose path ends with name, across all directories.""" + for dir_report in report.directories: + for verdict in dir_report.files: + if Path(verdict.path).name == name: + return verdict + raise AssertionError(f"no verdict for {name!r} in report") + + +# =========================================================================== +# TestMatchDecision - direct coverage of the reason-returning matchers +# =========================================================================== + + +class TestMatchDecision: + def test_dir_component_exclude_returns_pattern(self, tmp_path): + (tmp_path / ".git").mkdir() + source = _make_source(tmp_path, exclude=["**/.git/**"], include=[]) + matched, pattern = source._match_decision(tmp_path / ".git" / "x.pdf") + assert matched is False + assert pattern == "**/.git/**" + + def test_right_anchored_exclude_returns_pattern(self, tmp_path): + source = _make_source(tmp_path, exclude=["*.tmp"], include=[]) + matched, pattern = source._match_decision(tmp_path / "a.tmp") + assert (matched, pattern) == (False, "*.tmp") + + def test_exact_name_exclude_returns_pattern(self, tmp_path): + source = _make_source(tmp_path, exclude=[".DS_Store"], include=[]) + matched, pattern = source._match_decision(tmp_path / ".DS_Store") + assert (matched, pattern) == (False, ".DS_Store") + + def test_include_hit_returns_include_pattern(self, tmp_path): + source = _make_source(tmp_path, exclude=[], include=["**/*.pdf"]) + matched, pattern = source._match_decision(tmp_path / "doc.pdf") + assert (matched, pattern) == (True, "**/*.pdf") + + def test_include_miss_returns_none(self, tmp_path): + source = _make_source(tmp_path, exclude=[], include=["**/*.pdf"]) + matched, pattern = source._match_decision(tmp_path / "doc.xlsx") + assert (matched, pattern) == (False, None) + + def test_empty_include_passes_everything(self, tmp_path): + source = _make_source(tmp_path, exclude=[], include=[]) + matched, pattern = source._match_decision(tmp_path / "anything.bin") + assert (matched, pattern) == (True, None) + + def test_exclude_wins_over_include(self, tmp_path): + source = _make_source(tmp_path, exclude=["*.pdf"], include=["**/*.pdf"]) + matched, pattern = source._match_decision(tmp_path / "doc.pdf") + assert (matched, pattern) == (False, "*.pdf") + + def test_case_insensitive_by_default(self, tmp_path): + source = _make_source(tmp_path, exclude=[], include=["**/*.PDF"]) + matched, pattern = source._match_decision(tmp_path / "doc.pdf") + assert matched is True + # Pattern is echoed as written in config, not lowercased + assert pattern == "**/*.PDF" + + def test_case_sensitive_when_configured(self, tmp_path): + source = _make_source( + tmp_path, exclude=[], include=["**/*.PDF"], case_sensitive=True + ) + matched, pattern = source._match_decision(tmp_path / "doc.pdf") + assert (matched, pattern) == (False, None) + + def test_exclude_pattern_returned_in_original_case(self, tmp_path): + source = _make_source(tmp_path, exclude=["*.TMP"], include=[]) + matched, pattern = source._match_decision(tmp_path / "x.tmp") + assert (matched, pattern) == (False, "*.TMP") + + def test_directory_skip_pattern(self, tmp_path): + source = _make_source(tmp_path, exclude=["**/node_modules/**", "*.tmp"]) + assert source._directory_skip_pattern("node_modules") == "**/node_modules/**" + assert source._directory_skip_pattern("src") is None + # Non-dir-form excludes never prune traversal + assert source._directory_skip_pattern("a.tmp") is None + + def test_directory_skip_is_always_case_sensitive(self, tmp_path): + """Pins existing behavior: dir pruning ignores case_sensitive=False.""" + source = _make_source(tmp_path, exclude=["**/node_modules/**"]) + assert source.config.case_sensitive is False + assert source._directory_skip_pattern("NODE_MODULES") is None + + @pytest.mark.parametrize("name", ["doc.pdf", "doc.xlsx", "a.tmp", ".DS_Store"]) + def test_wrapper_parity(self, tmp_path, name): + """Bool wrappers must agree with the reason-returning variants.""" + source = _make_source( + tmp_path, exclude=["*.tmp", ".DS_Store"], include=["**/*.pdf"] + ) + path = tmp_path / name + assert source._matches_patterns(path) == source._match_decision(path)[0] + + @pytest.mark.parametrize("dir_name", ["node_modules", ".git", "src"]) + def test_dir_wrapper_parity(self, tmp_path, dir_name): + source = _make_source(tmp_path, exclude=["**/node_modules/**", "**/.git/**"]) + assert source._should_skip_directory(dir_name) == ( + source._directory_skip_pattern(dir_name) is not None + ) + + +# =========================================================================== +# TestScanEvents - the on_event callback on iter_candidates +# =========================================================================== + + +def _make_tree(tmp_path): + (tmp_path / ".git").mkdir() + (tmp_path / ".git" / "c.pdf").write_text("x") + (tmp_path / "a.pdf").write_text("x") + (tmp_path / "b.xlsx").write_text("x") + return _make_source( + tmp_path, include=["**/*.pdf"], exclude=["**/.git/**"] + ) + + +class TestScanEvents: + def test_candidates_identical_with_and_without_events(self, tmp_path): + """Drift guard: the callback must not change what gets scanned.""" + source = _make_tree(tmp_path) + plain = [c.path for c in source.iter_candidates()] + events = [] + observed = [c.path for c in source.iter_candidates(on_event=events.append)] + assert plain == observed + + def test_pruned_dir_and_excluded_file_events(self, tmp_path): + source = _make_tree(tmp_path) + events = [] + candidates = list(source.iter_candidates(on_event=events.append)) + + assert [Path(c.path).name for c in candidates] == ["a.pdf"] + by_kind = {e.kind: e for e in events} + assert by_kind["pruned_dir"].path == str((tmp_path / ".git").absolute()) + assert by_kind["pruned_dir"].pattern == "**/.git/**" + assert by_kind["excluded_file"].path == str((tmp_path / "b.xlsx").absolute()) + assert by_kind["excluded_file"].pattern is None # no include matched + + def test_exclude_pattern_carried_on_event(self, tmp_path): + (tmp_path / "x.tmp").write_text("x") + source = _make_source(tmp_path, include=[], exclude=["*.tmp"]) + events = [] + list(source.iter_candidates(on_event=events.append)) + assert events[0].kind == "excluded_file" + assert events[0].pattern == "*.tmp" + + def test_unreadable_file_event(self, tmp_path): + (tmp_path / "a.pdf").write_text("x") + source = _make_source(tmp_path, include=["**/*.pdf"], exclude=[]) + events = [] + with patch.object(LocalFileSystemSource, "_stat_candidate", return_value=None): + candidates = list(source.iter_candidates(on_event=events.append)) + assert candidates == [] + assert [e.kind for e in events] == ["unreadable_file"] + assert events[0].path == str((tmp_path / "a.pdf").absolute()) + + def test_non_recursive_events(self, tmp_path): + (tmp_path / "a.pdf").write_text("x") + (tmp_path / "b.xlsx").write_text("x") + source = _make_source( + tmp_path, include=["**/*.pdf"], exclude=[], recursive=False + ) + events = [] + candidates = list(source.iter_candidates(on_event=events.append)) + assert [Path(c.path).name for c in candidates] == ["a.pdf"] + assert [(e.kind, Path(e.path).name) for e in events] == [ + ("excluded_file", "b.xlsx") + ] + + +# =========================================================================== +# TestRunCheck - the filecheck orchestration +# =========================================================================== + + +class TestRunCheck: + def test_basic_verdicts_and_counts(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + (docs / ".git").mkdir() + (docs / ".git" / "hidden.pdf").write_text("x") + (docs / "a.pdf").write_text("x") + (docs / "b.xlsx").write_text("x") + config = _config_for(tmp_path, [ + {"path": str(docs), "include": ["**/*.pdf"], "exclude": ["**/.git/**"]} + ]) + + report = filecheck.run_check(config) + + assert report.error is None + assert _verdict_by_name(report, "a.pdf").status == "index" + assert "**/*.pdf" in _verdict_by_name(report, "a.pdf").reason + assert _verdict_by_name(report, "b.xlsx").status == "not_included" + dir_report = report.directories[0] + assert [p.pattern for p in dir_report.pruned_dirs] == ["**/.git/**"] + assert dir_report.counts["index"] == 1 + assert dir_report.counts["not_included"] == 1 + assert dir_report.counts["pruned_dirs"] == 1 + assert report.totals["index"] == 1 + assert filecheck.exit_code(report) == 0 + + def test_review_copy_and_superseded(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + old = docs / "Spec A.docx" + new = docs / "Spec B.docx" + review = docs / "Spec review.docx" + for f in (old, new, review): + f.write_text("x") + os.utime(old, (1000, 1000)) + os.utime(new, (2000, 2000)) + config = _config_for(tmp_path, [str(docs)]) + + report = filecheck.run_check(config) + + superseded = _verdict_by_name(report, "Spec A.docx") + assert superseded.status == "superseded" + assert superseded.latest_in_group == str(new.absolute()) + kept = _verdict_by_name(report, "Spec B.docx") + assert kept.status == "index" + assert kept.group == superseded.group + copy = _verdict_by_name(report, "Spec review.docx") + assert copy.status == "review_copy" + assert copy.pattern is not None + + def test_index_superseded_keeps_older_revisions(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + old = docs / "Spec A.docx" + new = docs / "Spec B.docx" + for f in (old, new): + f.write_text("x") + os.utime(old, (1000, 1000)) + os.utime(new, (2000, 2000)) + config = _config_for( + tmp_path, [str(docs)], versioning={"index_superseded": True} + ) + + report = filecheck.run_check(config) + + older = _verdict_by_name(report, "Spec A.docx") + assert older.status == "index" + assert "stamped superseded" in older.reason + + def test_disabled_directory_not_scanned(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + (docs / "a.pdf").write_text("x") + config = _config_for(tmp_path, [{"path": str(docs), "enabled": False}]) + + report = filecheck.run_check(config) + + dir_report = report.directories[0] + assert dir_report.enabled is False + assert dir_report.files == [] + assert filecheck.exit_code(report) == 0 # disabled is not an error + + def test_unavailable_directory_fails_exit_code(self, tmp_path): + config = _config_for(tmp_path, [str(tmp_path / "missing")]) + + report = filecheck.run_check(config) + + dir_report = report.directories[0] + assert dir_report.available is False + assert dir_report.unavailable_reason + assert filecheck.exit_code(report) == 1 + + def test_zero_directories_is_config_error(self, tmp_path): + config = { + "storage": {"storage_dir": str(tmp_path / "storage")}, + "indexing": {}, + } + + report = filecheck.run_check(config) + + assert report.error is not None + assert filecheck.exit_code(report) == 1 + + def test_duplicate_directory_reported(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + (docs / "a.pdf").write_text("x") + config = _config_for(tmp_path, [str(docs), str(docs)]) + + report = filecheck.run_check(config) + + first, second = report.directories + assert [v.status for v in first.files] == ["index"] + assert [v.status for v in second.files] == ["duplicate"] + assert str(docs.absolute()) in second.files[0].reason + + def test_non_recursive_skips_subdirectories(self, tmp_path): + docs = tmp_path / "docs" + (docs / "sub").mkdir(parents=True) + (docs / "top.md").write_text("x") + (docs / "sub" / "deep.md").write_text("x") + config = _config_for(tmp_path, [{"path": str(docs), "recursive": False}]) + + report = filecheck.run_check(config) + + names = [Path(v.path).name for v in report.directories[0].files] + assert "top.md" in names + assert "deep.md" not in names + assert report.directories[0].recursive is False + + def test_state_db_absent_and_never_created(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + (docs / "a.pdf").write_text("x") + config = _config_for(tmp_path, [str(docs)]) + state_db = tmp_path / "storage" / "ingestion_state.db" + + report = filecheck.run_check(config) + + assert report.state_db_present is False + assert _verdict_by_name(report, "a.pdf").indexed is None + # Read-only invariant: the check must not create storage or the DB + assert not state_db.exists() + assert not (tmp_path / "storage").exists() + + def test_state_db_presence_annotation(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + indexed_file = docs / "old.pdf" + new_file = docs / "new.pdf" + indexed_file.write_text("x") + new_file.write_text("x") + config = _config_for(tmp_path, [str(docs)]) + + (tmp_path / "storage").mkdir() + state = IngestionState(tmp_path / "storage" / "ingestion_state.db") + state.update_file_state( + FileInfo(path=str(indexed_file.absolute()), hash="h", last_modified=0.0), + ["doc1"], + ) + + report = filecheck.run_check(config) + + assert report.state_db_present is True + assert _verdict_by_name(report, "old.pdf").indexed is True + assert _verdict_by_name(report, "new.pdf").indexed is False + + def test_target_kept_file(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + (docs / "a.pdf").write_text("x") + config = _config_for(tmp_path, [str(docs)]) + + report = filecheck.run_check(config, target=str(docs / "a.pdf")) + + assert report.target is not None + assert report.target.status == "index" + assert filecheck.exit_code(report) == 0 + + def test_target_superseded_file(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + old = docs / "Spec A.docx" + new = docs / "Spec B.docx" + for f in (old, new): + f.write_text("x") + os.utime(old, (1000, 1000)) + os.utime(new, (2000, 2000)) + config = _config_for(tmp_path, [str(docs)]) + + report = filecheck.run_check(config, target=str(old)) + + assert report.target.status == "superseded" + assert report.target.latest_in_group == str(new.absolute()) + assert filecheck.exit_code(report) == 1 + + def test_target_under_pruned_directory(self, tmp_path): + docs = tmp_path / "docs" + (docs / ".git").mkdir(parents=True) + (docs / ".git" / "x.pdf").write_text("x") + config = _config_for(tmp_path, [str(docs)]) + + report = filecheck.run_check(config, target=str(docs / ".git" / "x.pdf")) + + assert report.target.status == "excluded" + assert "pruned" in report.target.reason + assert report.target.pattern == "**/.git/**" + assert filecheck.exit_code(report) == 1 + + def test_target_in_subdir_of_non_recursive_directory(self, tmp_path): + docs = tmp_path / "docs" + (docs / "sub").mkdir(parents=True) + (docs / "sub" / "deep.md").write_text("x") + config = _config_for(tmp_path, [{"path": str(docs), "recursive": False}]) + + report = filecheck.run_check(config, target=str(docs / "sub" / "deep.md")) + + assert report.target.status == "excluded" + assert "recursive: false" in report.target.reason + + def test_target_outside_configured_directories(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + config = _config_for(tmp_path, [str(docs)]) + + report = filecheck.run_check(config, target=str(tmp_path / "elsewhere.pdf")) + + assert report.target.status == "excluded" + assert "not under any configured" in report.target.reason + assert filecheck.exit_code(report) == 1 + + def test_target_nonexistent_file_under_directory(self, tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + config = _config_for(tmp_path, [str(docs)]) + + report = filecheck.run_check(config, target=str(docs / "ghost.pdf")) + + assert "does not exist" in report.target.reason + assert filecheck.exit_code(report) == 1 + + +# =========================================================================== +# TestCheckFilesCli - CLI wiring +# =========================================================================== + + +def _write_config(tmp_path, docs): + config_file = tmp_path / "config.yaml" + config_file.write_text( + f"storage:\n storage_dir: \"{tmp_path / 'storage'}\"\n" + f"indexing:\n directories:\n - \"{docs}\"\n" + ) + return config_file + + +class TestCheckFilesCli: + def _run_main(self, args): + from chunksilo.cli import main + + with patch("sys.argv", ["chunksilo"] + args): + with pytest.raises(SystemExit) as exc_info: + main() + return exc_info.value.code + + def test_full_scan_exit_zero(self, tmp_path, capsys): + docs = tmp_path / "docs" + docs.mkdir() + (docs / "a.md").write_text("x") + config_file = _write_config(tmp_path, docs) + + code = self._run_main(["--check-files", "--config", str(config_file)]) + + assert code == 0 + out = capsys.readouterr().out + assert "would index" in out + assert "a.md" in out + + def test_json_output(self, tmp_path, capsys): + docs = tmp_path / "docs" + docs.mkdir() + (docs / "a.md").write_text("x") + config_file = _write_config(tmp_path, docs) + + code = self._run_main( + ["--check-files", "--config", str(config_file), "--json"] + ) + + assert code == 0 + data = json.loads(capsys.readouterr().out) + assert data["totals"]["index"] == 1 + assert data["directories"][0]["files"][0]["status"] == "index" + + def test_skipped_target_exits_one(self, tmp_path, capsys): + docs = tmp_path / "docs" + docs.mkdir() + (docs / "notes.xlsx").write_text("x") + config_file = _write_config(tmp_path, docs) + + code = self._run_main([ + "--check-files", str(docs / "notes.xlsx"), "--config", str(config_file) + ]) + + assert code == 1 + assert "not included" in capsys.readouterr().out + + def test_unavailable_directory_exits_one(self, tmp_path): + config_file = _write_config(tmp_path, tmp_path / "missing") + + code = self._run_main(["--check-files", "--config", str(config_file)]) + + assert code == 1 + + def test_query_conflict_is_an_error(self, tmp_path): + config_file = _write_config(tmp_path, tmp_path) + + code = self._run_main( + ["some query", "--check-files", "--config", str(config_file)] + ) + + assert code == 2 # argparse error From 16dc162896a866d1f303e977e3f780f24dd13b0c Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:19:23 +0200 Subject: [PATCH 07/18] feat: write the MCP server log under storage_dir and load the config before anything else mcp.log was written to the current working directory, which for a service started from the wrong place can be inside an indexed tree, and rotated by renaming into mcp__.log files that accumulated forever. It now lives in storage.storage_dir, is created mode 0600, and rotates with a bounded backup count. Because the log location comes from the configuration, run_server now loads - and thereby activates - the configuration first, then configures logging, then imports the search module. The CHUNKSILO_CONFIG environment workaround that used to smuggle --config into the modules that read the config at import time is gone. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- src/chunksilo/server.py | 64 ++++++++++++++++----------- test/test_server.py | 95 ++++++++++++++++++++++++++--------------- 2 files changed, 98 insertions(+), 61 deletions(-) diff --git a/src/chunksilo/server.py b/src/chunksilo/server.py index e175273..3a20210 100644 --- a/src/chunksilo/server.py +++ b/src/chunksilo/server.py @@ -7,46 +7,53 @@ import argparse import asyncio import logging +import logging.handlers import os -from datetime import datetime from pathlib import Path from typing import Annotated, Any from mcp.server.fastmcp import FastMCP from pydantic import Field -# Log file configuration -LOG_FILE = "mcp.log" -LOG_MAX_SIZE_MB = 10 -LOG_MAX_SIZE_BYTES = LOG_MAX_SIZE_MB * 1024 * 1024 +# Log file configuration. The log lives in the storage directory - never the +# CWD, which on a mis-started service could be inside an indexed tree - and +# is private: it records what was searched. +LOG_FILE_NAME = "mcp.log" +LOG_MAX_SIZE_BYTES = 10 * 1024 * 1024 +LOG_BACKUP_COUNT = 5 # Module-level state (set during initialization) _server_config_path: Path | None = None _mcp: FastMCP | None = None -def _rotate_log_if_needed(): - """Rotate log file if it exists and is over the size limit.""" - log_path = Path(LOG_FILE) - if log_path.exists() and log_path.stat().st_size > LOG_MAX_SIZE_BYTES: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") - process_id = os.getpid() - rotated_name = f"mcp_{timestamp}_{process_id}.log" - rotated_path = log_path.parent / rotated_name - log_path.rename(rotated_path) - log_path.touch() +class _PrivateRotatingFileHandler(logging.handlers.RotatingFileHandler): + """RotatingFileHandler whose files are only ever mode 0600.""" + def _open(self): + fd = os.open( + self.baseFilename, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600 + ) + # O_CREAT does not chmod a file that already exists, so be explicit. + os.fchmod(fd, 0o600) + return os.fdopen(fd, self.mode, encoding=self.encoding) -def _setup_logging(): - """Configure logging for MCP server mode - file only, no stdout/stderr.""" - _rotate_log_if_needed() +def _setup_logging(storage_dir: str | Path): + """Configure logging for MCP server mode - file only, no stdout/stderr.""" + log_path = Path(storage_dir).expanduser() / LOG_FILE_NAME + log_path.parent.mkdir(parents=True, exist_ok=True) logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", force=True, handlers=[ - logging.FileHandler(LOG_FILE, encoding="utf-8"), + _PrivateRotatingFileHandler( + log_path, + maxBytes=LOG_MAX_SIZE_BYTES, + backupCount=LOG_BACKUP_COUNT, + encoding="utf-8", + ), ], ) logging.getLogger("llama_index.readers.confluence").setLevel(logging.WARNING) @@ -77,9 +84,20 @@ def run_server(config_path: Path | None = None): """Start the MCP server.""" global _server_config_path, _mcp + logger = logging.getLogger(__name__) + + from .cfgload import load_config + + # Load - and thereby activate - the configuration before importing + # .search/.index: index.py reads it at import time via load_config(). + config = load_config(config_path) if config_path: _server_config_path = config_path - os.environ["CHUNKSILO_CONFIG"] = str(config_path) + + # Logging goes to /mcp.log, so it could not be configured + # before the config was read. + _setup_logging(config["storage"]["storage_dir"]) + logger.info("Starting ChunkSilo MCP server") _mcp = _create_server() _mcp.run() @@ -96,12 +114,6 @@ def main(): parser.add_argument("--config", help="Path to config.yaml") args = parser.parse_args() - # Configure logging BEFORE importing anything that uses logging - _setup_logging() - - logger = logging.getLogger(__name__) - logger.info("Starting ChunkSilo MCP server") - config_path = Path(args.config) if args.config else None run_server(config_path) diff --git a/test/test_server.py b/test/test_server.py index ded1177..2e0a5f2 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -4,46 +4,71 @@ import asyncio from unittest.mock import patch -from chunksilo.server import _create_server, _rotate_log_if_needed +from chunksilo.server import _create_server, _setup_logging # ============================================================================= -# Tests for _rotate_log_if_needed +# Tests for _setup_logging # ============================================================================= -class TestRotateLogIfNeeded: - def test_oversized_log_rotated(self, tmp_path, monkeypatch): - log_file = tmp_path / "mcp.log" - # Write >10MB to the log file - log_file.write_bytes(b"x" * (11 * 1024 * 1024)) - - monkeypatch.setattr("chunksilo.server.LOG_FILE", str(log_file)) - _rotate_log_if_needed() - - # Original file should be recreated (empty) and a rotated file should exist - assert log_file.exists() - assert log_file.stat().st_size == 0 - rotated = [f for f in tmp_path.iterdir() if f.name.startswith("mcp_")] - assert len(rotated) == 1 - - def test_undersized_log_not_rotated(self, tmp_path, monkeypatch): - log_file = tmp_path / "mcp.log" - log_file.write_text("small log") - - monkeypatch.setattr("chunksilo.server.LOG_FILE", str(log_file)) - _rotate_log_if_needed() - - assert log_file.read_text() == "small log" - rotated = [f for f in tmp_path.iterdir() if f.name.startswith("mcp_")] - assert len(rotated) == 0 - - def test_nonexistent_log_no_error(self, tmp_path, monkeypatch): - log_file = tmp_path / "mcp.log" - monkeypatch.setattr("chunksilo.server.LOG_FILE", str(log_file)) - - # Should not raise - _rotate_log_if_needed() - assert not log_file.exists() +class TestSetupLogging: + def _teardown(self): + import logging + + # Release the file handler so tmp_path can be removed on Windows and + # later tests get a clean root logger. + for handler in logging.getLogger().handlers[:]: + logging.getLogger().removeHandler(handler) + handler.close() + + def test_log_lands_in_storage_dir_with_private_mode(self, tmp_path): + import logging + import stat + + try: + _setup_logging(tmp_path / "storage") + logging.getLogger("chunksilo.test").info("hello") + log_file = tmp_path / "storage" / "mcp.log" + assert log_file.exists() + assert stat.S_IMODE(log_file.stat().st_mode) == 0o600 + assert "hello" in log_file.read_text() + finally: + self._teardown() + + def test_existing_wider_mode_is_tightened(self, tmp_path): + import stat + + storage = tmp_path / "storage" + storage.mkdir() + log_file = storage / "mcp.log" + log_file.write_text("old") + log_file.chmod(0o644) + try: + _setup_logging(storage) + assert stat.S_IMODE(log_file.stat().st_mode) == 0o600 + finally: + self._teardown() + + def test_rotation_is_bounded(self, tmp_path): + import logging + + from chunksilo import server as server_module + + storage = tmp_path / "storage" + try: + with patch.object(server_module, "LOG_MAX_SIZE_BYTES", 512), patch.object( + server_module, "LOG_BACKUP_COUNT", 2 + ): + _setup_logging(storage) + log = logging.getLogger("chunksilo.test") + for _ in range(200): + log.info("x" * 64) + logs = sorted(p.name for p in storage.iterdir()) + assert "mcp.log" in logs + # backupCount bounds what rotation leaves behind + assert len(logs) <= 3 + finally: + self._teardown() # ============================================================================= From 08f1891b7dc5651ea2243a72924d879d61a1d780 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:25:08 +0200 Subject: [PATCH 08/18] perf: build result URIs from the stored path without touching the filesystem _resolve_file_uri called Path.exists() and Path.resolve() on every result path - for each BM25 candidate and for each formatted chunk, so 10-15 paths per query. resolve() readlinks every path component, and on a network mount that is a round trip per component, serialized through one userspace daemon: a single query against a few dozen documents could take the better part of a minute. It was also wrong twice over: resolve() rewrote a path the user recognises into the mount's internal form, which no application will open, and the URI was never percent-encoded, so any path containing a space produced a broken URI. The URI is now built from the stored path alone (a relative path is joined to the first configured directory) with quote(). Result URIs are consequently percent-encoded where they previously were not. The URI tests now exercise the real builder and pin the contract: no stat, readlink or resolve during URI building. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- src/chunksilo/search.py | 55 ++++++++++++++----- test/test_chunk_location.py | 103 +++++++++++++++++++++++++----------- 2 files changed, 115 insertions(+), 43 deletions(-) diff --git a/src/chunksilo/search.py b/src/chunksilo/search.py index 0605bf6..3cfb6b0 100644 --- a/src/chunksilo/search.py +++ b/src/chunksilo/search.py @@ -16,6 +16,7 @@ from datetime import datetime from pathlib import Path from typing import Any +from urllib.parse import quote from llama_index.core import Settings, StorageContext, load_index_from_storage from llama_index.core.schema import NodeWithScore, TextNode @@ -130,22 +131,50 @@ def _get_configured_directories(config: dict[str, Any]) -> list[Path]: return _configured_directories_cache -def _resolve_file_uri(file_path: str, config: dict[str, Any]) -> str | None: - """Resolve a file path to a file:// URI.""" - try: - file_path_obj = Path(str(file_path)) +def _normalize_result_path(file_path: str, config: dict[str, Any]) -> str | None: + """The stored path of a result, normalised - without touching the disk. + + This runs for every result of every query, so it must not hit the + filesystem. It used to call ``exists()`` and ``resolve()``, which on a + network mount (CIFS, or worse a GVFS/FUSE one) costs a stat plus a readlink + per path component - tens of round trips per result, serialized through one + userspace daemon, and the dominant cost of a query. + + Resolving was also wrong: it rewrote a path the user recognises into the + mount's internal form (``/run/user/1000/gvfs/smb-share:server=...``), which + no application will open. The stored path is what the indexer walked, so it + is used verbatim. + """ + raw = str(file_path) + if not raw: + return None + + if os.path.isabs(raw): + absolute = raw + else: + # Relative paths only appear for metadata written by older index + # versions. Attribute them to the first configured directory rather + # than probing the filesystem for the one that happens to hold them. + directories = _get_configured_directories(config) + base = str(directories[0]) if directories else "" + absolute = os.path.join(base, raw) if base else raw + + return os.path.normpath(absolute) - if file_path_obj.is_absolute(): - if file_path_obj.exists(): - return f"file://{file_path_obj.resolve()}" - return f"file://{file_path_obj}" - for data_dir in _get_configured_directories(config): - candidate = data_dir / file_path_obj - if candidate.exists(): - return f"file://{candidate.resolve()}" +def _file_uri(normalized: str) -> str: + # quote() must keep the separators; everything else (spaces, '#', '?', + # non-ASCII) has to be escaped or the URI silently truncates. + return "file://" + quote(normalized, safe="/") - return f"file://{file_path_obj.resolve()}" + +def _resolve_file_uri(file_path: str, config: dict[str, Any]) -> str | None: + """Build a ``file://`` URI for an indexed path, without touching the disk.""" + try: + normalized = _normalize_result_path(file_path, config) + if normalized is None: + return None + return _file_uri(normalized) except Exception: logger.debug("Failed to resolve file URI for %r", file_path, exc_info=True) return None diff --git a/test/test_chunk_location.py b/test/test_chunk_location.py index 00eb747..36da713 100644 --- a/test/test_chunk_location.py +++ b/test/test_chunk_location.py @@ -14,6 +14,11 @@ import pytest +from chunksilo import search +from chunksilo.search import _resolve_file_uri + +_NO_DIRS = {"indexing": {"directories": []}} + # ============================================================================= @@ -179,37 +184,77 @@ def test_char_start_before_first_heading(self): # ============================================================================= class TestURIBuilding: - """Tests for URI building logic in chunk processing""" + """Tests for the real URI builder used by the search pipeline. - def test_absolute_file_path(self): - """Test file:// URI generation for absolute paths""" - file_path = Path("/Users/test/data/document.pdf") - expected_uri = f"file://{file_path.resolve()}" + These exercise chunksilo.search._resolve_file_uri directly rather than a + copy of its logic, because the contract that matters cannot be restated: + it must not touch the filesystem. Every query formats up to fifteen result + paths, and on a network mount a stat plus a readlink per path component was + the dominant cost of a search. + """ - # Simulate the URI building logic - file_path_obj = Path(str(file_path)) - if file_path_obj.is_absolute(): - source_uri = f"file://{file_path_obj.resolve()}" - else: - source_uri = None + CONFIG = {"indexing": {"directories": ["/Users/test/data"]}} - assert source_uri == expected_uri + @pytest.fixture(autouse=True) + def _fresh_directory_cache(self, monkeypatch): + # The configured-directory list is cached module-wide; a test must not + # inherit another test's config. + monkeypatch.setattr(search, "_configured_directories_cache", None) - def test_relative_file_path(self): - """Test file:// URI generation for relative paths""" - with patch.dict(os.environ, {"DATA_DIR": "/Users/test/data"}): - file_path = "docs/readme.md" - data_dir = Path(os.getenv("DATA_DIR", "./data")) - - file_path_obj = Path(str(file_path)) - if file_path_obj.is_absolute(): - source_uri = f"file://{file_path_obj.resolve()}" - else: - resolved_path = (data_dir / file_path_obj).resolve() - source_uri = f"file://{resolved_path}" + def test_absolute_file_path(self): + """An absolute path is used verbatim, not canonicalised.""" + assert ( + _resolve_file_uri("/Users/test/data/document.pdf", self.CONFIG) + == "file:///Users/test/data/document.pdf" + ) - assert source_uri.startswith("file://") - assert "docs/readme.md" in source_uri or "readme.md" in source_uri + def test_relative_file_path(self): + """A relative path is joined to the first configured directory.""" + assert ( + _resolve_file_uri("docs/readme.md", self.CONFIG) + == "file:///Users/test/data/docs/readme.md" + ) + + def test_no_filesystem_access(self, monkeypatch): + """Nothing in URI building may stat, readlink or resolve.""" + def explode(*_args, **_kwargs): + raise AssertionError("URI building must not touch the filesystem") + + monkeypatch.setattr(os, "stat", explode) + monkeypatch.setattr(os, "lstat", explode) + monkeypatch.setattr(os.path, "realpath", explode) + monkeypatch.setattr(Path, "exists", explode) + monkeypatch.setattr(Path, "resolve", explode) + + assert ( + _resolve_file_uri("/mnt/docs/spec.pdf", self.CONFIG) + == "file:///mnt/docs/spec.pdf" + ) + + def test_spaces_and_reserved_characters_are_encoded(self): + """Real document paths are full of spaces; an unencoded URI is broken.""" + uri = _resolve_file_uri( + "/srv/docs/Widget Project/10 - Design Notes/Rev A/a#b.docx", + self.CONFIG, + ) + assert uri == ( + "file:///srv/docs/Widget%20Project/10%20-%20Design%20Notes/" + "Rev%20A/a%23b.docx" + ) + + def test_mount_internal_path_is_not_substituted(self): + """A symlinked mount must keep the name the user knows it by. + + resolve() used to rewrite ~/nas/... into + /run/user/1000/gvfs/smb-share:server=..., which no application opens. + """ + assert ( + _resolve_file_uri("/home/alice/nas/Projects/spec.docx", self.CONFIG) + == "file:///home/alice/nas/Projects/spec.docx" + ) + + def test_empty_path(self): + assert _resolve_file_uri("", self.CONFIG) is None def test_confluence_uri_with_page_id(self): """Test Confluence URL generation with page_id""" @@ -300,8 +345,7 @@ def test_pdf_chunk_location(self): headings = metadata.get("document_headings", []) # Build URI - file_path_obj = Path(str(file_path)) - source_uri = f"file://{file_path_obj.resolve()}" + source_uri = _resolve_file_uri(str(file_path), _NO_DIRS) # Get page page = metadata.get("page_label") or metadata.get("page_number") or metadata.get("page") @@ -341,8 +385,7 @@ def test_markdown_chunk_location(self): line_offsets = metadata.get("line_offsets") # Build URI - file_path_obj = Path(str(file_path)) - source_uri = f"file://{file_path_obj.resolve()}" + source_uri = _resolve_file_uri(str(file_path), _NO_DIRS) # Get page (None for markdown) page = None From 0f20a8e4f488a6c2d655bd2376d38e35304e0a07 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:29:59 +0200 Subject: [PATCH 09/18] feat: present files from mounted network shares at their share location Search results carried file:// URIs built from the indexer's own mount paths, which a client on another machine can neither open nor act on. A new `shares` configuration lists where each mounted tree really lives (local prefix -> //server/share), and results for covered files now name that location: an smb:// URI plus the Windows UNC path, in a new `unc` field next to `uri`. The mapping is derived purely from configuration with component-anchored, post-normpath matching (longest prefix wins, and `..` cannot cross into another share), and like the rest of result formatting it never touches the filesystem on the query path. Files not covered by any mapping - including everything in a local single-machine setup, which typically configures none - keep their file:// URIs and a null `unc`. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- config.yaml | 8 +++ src/chunksilo/cfgload.py | 6 ++ src/chunksilo/search.py | 50 +++++++++++++- src/chunksilo/shareuri.py | 109 +++++++++++++++++++++++++++++ test/test_chunk_location.py | 33 +++++++++ test/test_share_uri.py | 132 ++++++++++++++++++++++++++++++++++++ 6 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 src/chunksilo/shareuri.py create mode 100644 test/test_share_uri.py diff --git a/config.yaml b/config.yaml index 19533c3..94ed2aa 100644 --- a/config.yaml +++ b/config.yaml @@ -104,3 +104,11 @@ retrieval: storage: storage_dir: "./storage" model_cache_dir: "./models" + +# Network shares (optional): present files indexed from a mounted share at the +# share's own location - an smb:// URI plus the Windows UNC path - instead of +# the local mount path, so clients on other machines get a location they can +# open. Longest matching prefix wins; files under no prefix keep file:// URIs. +# shares: +# - prefix: "/mnt/docs" # local mount point as listed in indexing.directories +# unc: "//nas/docs" # the same tree as clients reach it diff --git a/src/chunksilo/cfgload.py b/src/chunksilo/cfgload.py index 0703b78..23d5e76 100644 --- a/src/chunksilo/cfgload.py +++ b/src/chunksilo/cfgload.py @@ -155,6 +155,12 @@ def _find_config() -> Path: "storage_dir": "./storage", "model_cache_dir": "./models", }, + # Present files indexed from a mounted network share at the share's own + # location (an smb:// URI plus the Windows UNC path) instead of the + # server-local mount path, so a client on another machine gets a location + # it can open. Uncovered files keep their file:// URIs. + # [{prefix: "/mnt/docs", unc: "//nas/docs"}] + "shares": [], } # Options that no longer exist, mapped to what replaced them. A config file diff --git a/src/chunksilo/search.py b/src/chunksilo/search.py index 3cfb6b0..40a805c 100644 --- a/src/chunksilo/search.py +++ b/src/chunksilo/search.py @@ -48,6 +48,7 @@ from .cfgload import load_config from .models import _get_cached_model_path, configure_offline_mode, resolve_flashrank_model_name +from .shareuri import build_share_mappings, to_share_uris logger = logging.getLogger(__name__) @@ -180,6 +181,44 @@ def _resolve_file_uri(file_path: str, config: dict[str, Any]) -> str | None: return None +# Keyed by config object identity: the server holds one config for its +# lifetime, so this builds once; a test handing in a fresh dict just rebuilds. +_share_mappings_cache: tuple[Any, tuple] | None = None + + +def _get_share_mappings(config: dict[str, Any]): + global _share_mappings_cache + cached = _share_mappings_cache + if cached is not None and cached[0] is config: + return cached[1] + mappings = build_share_mappings(config) + _share_mappings_cache = (config, mappings) + return mappings + + +def _resolve_result_uris( + file_path: str, config: dict[str, Any] +) -> tuple[str | None, str | None]: + """``(uri, unc)`` to present for a result path - never touching the disk. + + A file covered by a ``shares`` mapping is presented as the share location + the user actually knows: an ``smb://`` URI plus the Windows UNC form. + Everything else keeps the local ``file://`` URI, which is exactly right + for a same-machine client. + """ + try: + normalized = _normalize_result_path(file_path, config) + if normalized is None: + return None, None + share = to_share_uris(normalized, _get_share_mappings(config)) + if share is not None: + return share + return _file_uri(normalized), None + except Exception: + logger.debug("Failed to resolve result URIs for %r", file_path, exc_info=True) + return None, None + + def _build_heading_path(headings: list[dict], char_start: int | None) -> tuple[str | None, list[str]]: """Build a heading path for the given character position within a document.""" if not headings or char_start is None: @@ -313,9 +352,12 @@ def _format_bm25_matches(bm25_nodes: list[NodeWithScore], config: dict[str, Any] continue metadata = node.node.metadata or {} file_path = metadata.get("file_path", "") - source_uri = _resolve_file_uri(file_path, config) if file_path else None + source_uri, source_unc = ( + _resolve_result_uris(file_path, config) if file_path else (None, None) + ) matched_files.append({ "uri": source_uri, + "unc": source_unc, "score": round(float(node.score), 4), }) return matched_files[:5] @@ -1309,8 +1351,9 @@ def _format_search_results( if heading_text and (not heading_path or heading_path[-1] != heading_text): heading_path = heading_path + [heading_text] if heading_path else [heading_text] - # Build URI + # Build URI (plus the Windows UNC form for share-covered files) source_uri = None + source_unc = None if original_source == "Confluence": confluence_url = config["confluence"]["url"] page_id = metadata.get("page_id") @@ -1328,7 +1371,7 @@ def _format_search_results( if jira_url and issue_key: source_uri = f"{jira_url.rstrip('/')}/browse/{issue_key}" elif file_path: - source_uri = _resolve_file_uri(file_path, config) + source_uri, source_unc = _resolve_result_uris(file_path, config) page_number = ( metadata.get("page_label") @@ -1343,6 +1386,7 @@ def _format_search_results( location = { "uri": source_uri, + "unc": source_unc, "page": page_number, "line": line_number, "heading_path": heading_path if heading_path else None, diff --git a/src/chunksilo/shareuri.py b/src/chunksilo/shareuri.py new file mode 100644 index 0000000..f5711ef --- /dev/null +++ b/src/chunksilo/shareuri.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Present indexed paths at the network-share location users actually know. + +Search results carry ``file://`` URIs built from the path the *indexer* +walked. When that path is a mount point on the machine running ChunkSilo, it +means nothing to a client on another machine. The ``shares`` configuration +states where each mounted tree really lives (``prefix`` -> ``unc``), so +results for covered files can name the share itself: an ``smb://`` URI plus +the Windows UNC form of the same location. + +Everything here is pure string work over configuration: like the rest of the +result-formatting path it must never touch the filesystem (AGENTS.md, Search +Invariants). The mapping is matched component-wise after ``normpath``, so +``..`` cannot carry one share's mapping across into another's. +""" +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from typing import Any +from urllib.parse import quote + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ShareMapping: + """One local-mount -> share mapping from the ``shares`` configuration.""" + + prefix: str + prefix_parts: tuple[str, ...] + host: str + share_parts: tuple[str, ...] + + +def _path_parts(path: str) -> list[str]: + """Path components after normalisation, with empties and ``.`` dropped. + + normpath collapses ``.`` and ``..`` first, so a path cannot climb out of + its share and still match. Any ``..`` that survives (a relative path + climbing above its own root) is kept as a component precisely so it cannot + match a real prefix. + """ + return [ + part for part in os.path.normpath(path).split(os.sep) if part and part != "." + ] + + +def build_share_mappings(config: dict[str, Any]) -> tuple[ShareMapping, ...]: + """Mappings from the ``shares`` list, longest prefix first. + + Invalid entries are skipped with a warning rather than raised: a typo must + degrade that entry's URIs to the ``file://`` fallback, never break search. + """ + mappings: list[ShareMapping] = [] + for rule in config.get("shares") or []: + if not isinstance(rule, dict): + continue + prefix = str(rule.get("prefix") or "").strip() + unc = str(rule.get("unc") or "").strip().replace("\\", "/") + prefix_parts = tuple(_path_parts(prefix)) if prefix else () + unc_parts = [part for part in unc.split("/") if part] + if ( + not os.path.isabs(prefix) + or not prefix_parts # a "/" prefix would map the whole filesystem + or not unc.startswith("//") + or len(unc_parts) < 2 # need at least //server/share + ): + logger.warning( + "Ignoring shares entry: prefix must be an absolute path and unc " + "must look like //server/share (prefix=%r, unc=%r)", + rule.get("prefix"), rule.get("unc"), + ) + continue + mappings.append( + ShareMapping( + prefix=os.path.normpath(prefix), + prefix_parts=prefix_parts, + host=unc_parts[0], + share_parts=tuple(unc_parts[1:]), + ) + ) + mappings.sort(key=lambda mapping: len(mapping.prefix_parts), reverse=True) + return tuple(mappings) + + +def to_share_uris( + normalized_path: str, mappings: tuple[ShareMapping, ...] +) -> tuple[str, str] | None: + """``(smb:// URI, UNC path)`` for a covered path, or None. + + The ``smb://`` form percent-encodes each segment (spaces, ``#`` and ``?`` + would silently truncate the URI); the UNC form is raw, because Windows + opens ``\\\\server\\share\\...`` verbatim and does not understand percent + escapes. + """ + if not mappings: + return None + parts = _path_parts(normalized_path) + for mapping in mappings: + if parts[: len(mapping.prefix_parts)] == list(mapping.prefix_parts): + segments = list(mapping.share_parts) + parts[len(mapping.prefix_parts):] + smb = "smb://" + mapping.host + "/" + "/".join( + quote(segment, safe="") for segment in segments + ) + unc = "\\\\" + mapping.host + "\\" + "\\".join(segments) + return smb, unc + return None diff --git a/test/test_chunk_location.py b/test/test_chunk_location.py index 36da713..340165c 100644 --- a/test/test_chunk_location.py +++ b/test/test_chunk_location.py @@ -256,6 +256,39 @@ def test_mount_internal_path_is_not_substituted(self): def test_empty_path(self): assert _resolve_file_uri("", self.CONFIG) is None + SHARE_CONFIG = { + "indexing": {"directories": []}, + "shares": [{"prefix": "/mnt/docs", "unc": "//nas/docs"}], + } + + def test_share_mapping_no_filesystem_access(self, monkeypatch): + """The share-location mapping is pure config string work; it may not + stat either - it runs for the same fifteen results per query.""" + def explode(*_args, **_kwargs): + raise AssertionError("URI building must not touch the filesystem") + + monkeypatch.setattr(os, "stat", explode) + monkeypatch.setattr(os, "lstat", explode) + monkeypatch.setattr(os.path, "realpath", explode) + monkeypatch.setattr(Path, "exists", explode) + monkeypatch.setattr(Path, "resolve", explode) + + from chunksilo.search import _resolve_result_uris + + assert _resolve_result_uris("/mnt/docs/spec.pdf", self.SHARE_CONFIG) == ( + "smb://nas/docs/spec.pdf", + "\\\\nas\\docs\\spec.pdf", + ) + + def test_uncovered_path_keeps_the_file_uri_exactly(self): + """No share covering a path: the file:// URI, byte-for-byte.""" + from chunksilo.search import _resolve_result_uris + + assert _resolve_result_uris("/opt/notes/spec.pdf", self.SHARE_CONFIG) == ( + "file:///opt/notes/spec.pdf", + None, + ) + def test_confluence_uri_with_page_id(self): """Test Confluence URL generation with page_id""" with patch.dict(os.environ, {"CONFLUENCE_URL": "https://wiki.example.com"}): diff --git a/test/test_share_uri.py b/test/test_share_uri.py new file mode 100644 index 0000000..2738325 --- /dev/null +++ b/test/test_share_uri.py @@ -0,0 +1,132 @@ +"""Tests for share-location presentation of search results (shareuri.py). + +Results for files indexed from a mounted network share must name the location +the user actually knows (smb:// plus the Windows UNC form), derived purely +from the ``shares`` config - never from the mount table, and never by touching +the filesystem. +""" +import logging + +from llama_index.core.schema import NodeWithScore, TextNode + +from chunksilo import search +from chunksilo.search import _format_bm25_matches, _resolve_result_uris +from chunksilo.shareuri import build_share_mappings, to_share_uris + +DOCS = {"prefix": "/mnt/docs", "unc": "//nas/docs"} + + +def _mappings(*rules): + return build_share_mappings({"shares": list(rules)}) + + +class TestBuildShareMappings: + def test_valid_entry(self): + (mapping,) = _mappings(DOCS) + assert mapping.host == "nas" + assert mapping.share_parts == ("docs",) + assert mapping.prefix_parts == ("mnt", "docs") + + def test_nested_share_path(self): + (mapping,) = _mappings({"prefix": "/mnt/eng", "unc": "//nas/dept/engineering"}) + assert mapping.host == "nas" + assert mapping.share_parts == ("dept", "engineering") + + def test_windows_style_unc_is_normalized(self): + (mapping,) = _mappings({"prefix": "/mnt/x", "unc": "\\\\nas\\docs"}) + assert mapping.host == "nas" + assert mapping.share_parts == ("docs",) + + def test_invalid_entries_are_skipped_with_a_warning(self, caplog): + """A typo must degrade that entry's URIs, never break search.""" + with caplog.at_level(logging.WARNING, logger="chunksilo.shareuri"): + mappings = _mappings( + {"prefix": "relative/path", "unc": "//h/s"}, + {"prefix": "/ok", "unc": "not-a-unc"}, + {"prefix": "/", "unc": "//h/s"}, # would map the whole filesystem + {"prefix": "/mnt/x", "unc": "//hostonly"}, # no share component + DOCS, + ) + assert len(mappings) == 1 + assert mappings[0].prefix == "/mnt/docs" + assert caplog.text.count("Ignoring shares entry") == 4 + + def test_sorted_longest_prefix_first(self): + mappings = _mappings( + {"prefix": "/mnt", "unc": "//h/top"}, + {"prefix": "/mnt/docs", "unc": "//h/deep"}, + ) + assert [m.share_parts for m in mappings] == [("deep",), ("top",)] + + +class TestToShareUris: + def test_covered_path(self): + smb, unc = to_share_uris("/mnt/docs/spec.pdf", _mappings(DOCS)) + assert smb == "smb://nas/docs/spec.pdf" + assert unc == "\\\\nas\\docs\\spec.pdf" + + def test_smb_is_encoded_and_unc_is_raw(self): + """Windows opens UNC verbatim; percent escapes belong in the URI only.""" + smb, unc = to_share_uris("/mnt/docs/10 - Design Notes/a#b.docx", _mappings(DOCS)) + assert smb == "smb://nas/docs/10%20-%20Design%20Notes/a%23b.docx" + assert unc == "\\\\nas\\docs\\10 - Design Notes\\a#b.docx" + + def test_longest_prefix_wins(self): + mappings = _mappings( + {"prefix": "/mnt", "unc": "//h/broad"}, + {"prefix": "/mnt/docs", "unc": "//h/narrow"}, + ) + smb, _ = to_share_uris("/mnt/docs/f.txt", mappings) + assert smb == "smb://h/narrow/f.txt" + smb, _ = to_share_uris("/mnt/other/f.txt", mappings) + assert smb == "smb://h/broad/other/f.txt" + + def test_dotdot_cannot_cross_into_a_share(self): + """normpath moves the path out of the covered tree before matching.""" + assert to_share_uris("/mnt/docs/../secrets/f.txt", _mappings(DOCS)) is None + + def test_prefix_matching_is_component_anchored(self): + assert to_share_uris("/mnt/docs-archive/f.txt", _mappings(DOCS)) is None + + def test_uncovered_path(self): + assert to_share_uris("/opt/docs/f.txt", _mappings(DOCS)) is None + assert to_share_uris("/opt/docs/f.txt", ()) is None + + +class TestSearchIntegration: + CONFIG = {"indexing": {"directories": []}, "shares": [DOCS]} + + def test_share_covered_result(self): + assert _resolve_result_uris("/mnt/docs/spec.pdf", self.CONFIG) == ( + "smb://nas/docs/spec.pdf", + "\\\\nas\\docs\\spec.pdf", + ) + + def test_uncovered_result_keeps_the_exact_file_uri(self): + config = {"indexing": {"directories": []}, "shares": []} + assert _resolve_result_uris("/opt/docs/read me.md", config) == ( + "file:///opt/docs/read%20me.md", + None, + ) + + def test_relative_path_joins_the_first_directory_then_maps(self, monkeypatch): + monkeypatch.setattr(search, "_configured_directories_cache", None) + config = {"indexing": {"directories": ["/mnt/docs"]}, "shares": [DOCS]} + assert _resolve_result_uris("guides/spec.pdf", config) == ( + "smb://nas/docs/guides/spec.pdf", + "\\\\nas\\docs\\guides\\spec.pdf", + ) + + def test_bm25_matches_carry_uri_and_unc(self): + node = TextNode( + text="x", + metadata={"file_path": "/mnt/docs/a.pdf", "file_name": "a.pdf"}, + ) + (entry,) = _format_bm25_matches( + [NodeWithScore(node=node, score=1.2)], self.CONFIG + ) + assert entry == { + "uri": "smb://nas/docs/a.pdf", + "unc": "\\\\nas\\docs\\a.pdf", + "score": 1.2, + } From 507c438567d9915bd4782adb079b0bdcc900a5b9 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:35:40 +0200 Subject: [PATCH 10/18] perf: load search models once under concurrency and log per-stage timings The four lazy loaders (index, embedding model, reranker, BM25) were unguarded module globals. A server that dispatches tool calls into a thread pool with no session affinity lets N concurrent first calls each cold-start their own copy - N times the work and memory, N-1 discarded - and every one of them waits for the slowest. They are now double-checked-locked, the reranker is published only after it loaded successfully (a failed load is retried, not cached), and a new search.warm_up() lets a server pay the cost once at startup instead of inside a user's query. Per-stage timings (load, vector, bm25, remote, filter, rerank, format) are logged for every search, because "the query took 56 seconds" is not actionable on its own. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- src/chunksilo/search.py | 229 ++++++++++++++++++++++---------- test/test_search_concurrency.py | 169 +++++++++++++++++++++++ 2 files changed, 327 insertions(+), 71 deletions(-) create mode 100644 test/test_search_concurrency.py diff --git a/src/chunksilo/search.py b/src/chunksilo/search.py index 40a805c..1a6101b 100644 --- a/src/chunksilo/search.py +++ b/src/chunksilo/search.py @@ -10,6 +10,7 @@ import math import os import re +import threading import time from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FuturesTimeoutError @@ -71,10 +72,20 @@ def _get_config() -> dict[str, Any]: # Global caches +# Global caches. Each is guarded by its own lock: an HTTP server dispatches +# every tool call into a thread pool with no session affinity, so concurrent +# first requests would otherwise each cold-start their own index load, ONNX +# session and reranker - three times the work and three times the memory, with +# two of each thrown away. Loading a model takes seconds, so the second caller +# waiting on the lock is far cheaper than duplicating it. _index_cache = None +_index_lock = threading.Lock() _embed_model_initialized = False +_embed_model_lock = threading.Lock() _reranker_model = None +_reranker_lock = threading.Lock() _bm25_retriever_cache = None +_bm25_lock = threading.Lock() _configured_directories_cache: list[Path] | None = None # Common English stopwords to filter from Confluence CQL queries @@ -257,91 +268,106 @@ def _char_offset_to_line(char_offset: int | None, line_offsets: list[int] | None def _ensure_embed_model(config: dict[str, Any]) -> None: - """Ensure the embedding model is initialized.""" + """Ensure the embedding model is initialized (once, even under concurrency).""" global _embed_model_initialized if _embed_model_initialized: return - model_name = config["retrieval"]["embed_model_name"] - cache_dir = Path(config["storage"]["model_cache_dir"]) - offline_mode = config["retrieval"]["offline"] - - cached_model_path = _get_cached_model_path(cache_dir, model_name) - if cached_model_path and offline_mode: - logger.info(f"Loading embedding model from cache: {cached_model_path}") - embed_model = FastEmbedEmbedding( - model_name=model_name, - cache_dir=str(cache_dir), - specific_model_path=str(cached_model_path) - ) - else: - embed_model = FastEmbedEmbedding( - model_name=model_name, - cache_dir=str(cache_dir), - ) - logger.info("Embedding model initialized successfully") - Settings.embed_model = embed_model - _embed_model_initialized = True + with _embed_model_lock: + if _embed_model_initialized: + return + + model_name = config["retrieval"]["embed_model_name"] + cache_dir = Path(config["storage"]["model_cache_dir"]) + offline_mode = config["retrieval"]["offline"] + + cached_model_path = _get_cached_model_path(cache_dir, model_name) + if cached_model_path and offline_mode: + logger.info(f"Loading embedding model from cache: {cached_model_path}") + embed_model = FastEmbedEmbedding( + model_name=model_name, + cache_dir=str(cache_dir), + specific_model_path=str(cached_model_path) + ) + else: + embed_model = FastEmbedEmbedding( + model_name=model_name, + cache_dir=str(cache_dir), + ) + logger.info("Embedding model initialized successfully") + Settings.embed_model = embed_model + _embed_model_initialized = True def _ensure_reranker(config: dict[str, Any]): - """Load the FlashRank reranking model.""" + """Load the FlashRank reranking model (once, even under concurrency).""" global _reranker_model if _reranker_model is not None: return _reranker_model - try: - from flashrank import Ranker - except ImportError as exc: - raise ImportError( - "flashrank is required for reranking. Install with: pip install chunksilo" - ) from exc + with _reranker_lock: + if _reranker_model is not None: + return _reranker_model - raw_model_name = config["retrieval"]["rerank_model_name"] - cache_dir = Path(config["storage"]["model_cache_dir"]) - offline_mode = config["retrieval"]["offline"] + try: + from flashrank import Ranker + except ImportError as exc: + raise ImportError( + "flashrank is required for reranking. Install with: pip install chunksilo" + ) from exc - model_name = resolve_flashrank_model_name(raw_model_name) + raw_model_name = config["retrieval"]["rerank_model_name"] + cache_dir = Path(config["storage"]["model_cache_dir"]) + offline_mode = config["retrieval"]["offline"] - try: - _reranker_model = Ranker(model_name=model_name, cache_dir=str(cache_dir)) - except Exception as exc: - if offline_mode: - raise FileNotFoundError( - f"Rerank model '{model_name}' not available in cache directory {cache_dir}. " - "Download it before running in offline mode." - ) from exc - raise + model_name = resolve_flashrank_model_name(raw_model_name) - logger.info(f"Rerank model '{model_name}' loaded successfully") - return _reranker_model + try: + ranker = Ranker(model_name=model_name, cache_dir=str(cache_dir)) + except Exception as exc: + if offline_mode: + raise FileNotFoundError( + f"Rerank model '{model_name}' not available in cache directory " + f"{cache_dir}. Download it before running in offline mode." + ) from exc + raise + + logger.info(f"Rerank model '{model_name}' loaded successfully") + _reranker_model = ranker + return _reranker_model def _ensure_bm25_retriever(config: dict[str, Any]): - """Load the BM25 retriever for file name matching.""" + """Load the BM25 retriever for file name matching (once, even concurrently).""" global _bm25_retriever_cache if _bm25_retriever_cache is not None: return _bm25_retriever_cache - storage_dir = Path(config["storage"]["storage_dir"]) - bm25_index_dir = storage_dir / "bm25_index" + with _bm25_lock: + if _bm25_retriever_cache is not None: + return _bm25_retriever_cache - if not bm25_index_dir.exists(): - logger.warning(f"BM25 index not found at {bm25_index_dir}. Run indexing to create it.") - return None + storage_dir = Path(config["storage"]["storage_dir"]) + bm25_index_dir = storage_dir / "bm25_index" - try: - from llama_index.retrievers.bm25 import BM25Retriever - logger.info(f"Loading BM25 index from {bm25_index_dir}") - _bm25_retriever_cache = BM25Retriever.from_persist_dir(str(bm25_index_dir)) - logger.info("BM25 retriever loaded successfully") - return _bm25_retriever_cache - except Exception as e: - logger.error(f"Failed to load BM25 retriever: {e}") - return None + if not bm25_index_dir.exists(): + logger.warning( + f"BM25 index not found at {bm25_index_dir}. Run indexing to create it." + ) + return None + + try: + from llama_index.retrievers.bm25 import BM25Retriever + logger.info(f"Loading BM25 index from {bm25_index_dir}") + _bm25_retriever_cache = BM25Retriever.from_persist_dir(str(bm25_index_dir)) + logger.info("BM25 retriever loaded successfully") + return _bm25_retriever_cache + except Exception as e: + logger.error(f"Failed to load BM25 retriever: {e}") + return None def _format_bm25_matches(bm25_nodes: list[NodeWithScore], config: dict[str, Any]) -> list[dict[str, Any]]: @@ -1019,7 +1045,7 @@ def _search_jira(query: str, config: dict[str, Any]) -> list[NodeWithScore]: def load_llamaindex_index(config: dict[str, Any] | None = None): - """Load the LlamaIndex from storage.""" + """Load the LlamaIndex from storage (once, even under concurrency).""" if config is None: config = _get_config() global _index_cache @@ -1027,19 +1053,56 @@ def load_llamaindex_index(config: dict[str, Any] | None = None): if _index_cache is not None: return _index_cache - storage_dir = Path(config["storage"]["storage_dir"]) - if not storage_dir.exists(): - raise FileNotFoundError( - f"Storage directory {storage_dir} does not exist. " - "Please run indexing first." - ) + with _index_lock: + if _index_cache is not None: + return _index_cache + + storage_dir = Path(config["storage"]["storage_dir"]) + if not storage_dir.exists(): + raise FileNotFoundError( + f"Storage directory {storage_dir} does not exist. " + "Please run indexing first." + ) + + logger.info("Loading LlamaIndex from storage...") + _ensure_embed_model(config) + storage_context = StorageContext.from_defaults(persist_dir=str(storage_dir)) + index = load_index_from_storage(storage_context) + _index_cache = index + return index + + +def warm_up(config: dict[str, Any] | None = None) -> None: + """Load everything a query needs, before any query arrives. + + A server can call this at startup. Without it the first requests pay for + the index load, the embedding ONNX session, the reranker and the BM25 index - + and concurrent first requests all wait on whichever of them got there first, + which is how a few parallel tool calls end up past a client's timeout. + Failures are logged, not raised: a server that cannot preload is still + worth starting, and the same error will surface per query. + """ + if config is None: + config = _get_config() - logger.info("Loading LlamaIndex from storage...") - _ensure_embed_model(config) - storage_context = StorageContext.from_defaults(persist_dir=str(storage_dir)) - index = load_index_from_storage(storage_context) - _index_cache = index - return index + configure_offline_mode( + config["retrieval"]["offline"], + Path(config["storage"]["model_cache_dir"]), + ) + + started = time.time() + for name, load in ( + ("index", lambda: load_llamaindex_index(config)), + ("embedding model", lambda: _ensure_embed_model(config)), + ("reranker", lambda: _ensure_reranker(config)), + ("BM25 index", lambda: _ensure_bm25_retriever(config)), + ("heading store", get_heading_store), + ): + try: + load() + except Exception as exc: + logger.error("Warm-up of the %s failed: %s", name, exc) + logger.info("Warm-up finished in %.1fs", time.time() - started) def _parse_iso8601_to_date(iso_string: str) -> str | None: @@ -1431,17 +1494,32 @@ def run_search( _setup_ssl(config) start_time = time.time() + # Per-stage timings: without them, "the query took 56 seconds" is + # unactionable. Logged rather than returned, so the tool response shape + # is unchanged. + stage_times: dict[str, float] = {} + stage_start = start_time + + def _mark(stage: str) -> None: + nonlocal stage_start + now = time.time() + stage_times[stage] = now - stage_start + stage_start = now try: required_phrases, clean_query = _extract_quoted_phrases(query) enhanced_query = _preprocess_query(clean_query) index = load_llamaindex_index(config) + _mark("load") # Retrieval nodes = _retrieve_vector_nodes(index, enhanced_query, required_phrases, config) + _mark("vector") matched_files = _retrieve_bm25_matches(enhanced_query, required_phrases, config) + _mark("bm25") nodes.extend(_retrieve_remote_sources(enhanced_query, config)) + _mark("remote") # Date filtering and recency boost if date_from or date_to: @@ -1460,8 +1538,11 @@ def run_search( logger.info(f"Capping rerank candidates: {len(nodes)} -> {rerank_candidates}") nodes = nodes[:rerank_candidates] + _mark("filter") + # Rerank nodes, rerank_scores = _rerank_nodes(nodes, enhanced_query, config) + _mark("rerank") # Post-rerank filtering score_threshold = config["retrieval"]["score_threshold"] @@ -1482,7 +1563,13 @@ def run_search( # Format results chunks = _format_search_results(nodes, rerank_scores, config) + _mark("format") elapsed = time.time() - start_time + logger.info( + "Search took %.2fs (%s)", + elapsed, + " ".join(f"{name}={secs:.2f}s" for name, secs in stage_times.items()), + ) return { "matched_files": matched_files, diff --git a/test/test_search_concurrency.py b/test/test_search_concurrency.py new file mode 100644 index 0000000..c6c19a9 --- /dev/null +++ b/test/test_search_concurrency.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Concurrent first-use of the search pipeline must not duplicate cold starts. + +An HTTP server dispatches every tool call into a thread pool with no session +affinity, so several clients can arrive before anything is loaded. Each loader +used to be an unguarded module global, so N simultaneous first calls each built +their own index, ONNX session and reranker - N times the work and memory, with +N-1 of each discarded. Three parallel calls that way exceed a 60-second client +timeout on a cold server. +""" +import threading +import time + +import pytest + +from chunksilo import search + + +@pytest.fixture(autouse=True) +def clean_caches(monkeypatch): + """Start every test with an unloaded pipeline and restore afterwards.""" + monkeypatch.setattr(search, "_index_cache", None) + monkeypatch.setattr(search, "_embed_model_initialized", False) + monkeypatch.setattr(search, "_reranker_model", None) + monkeypatch.setattr(search, "_bm25_retriever_cache", None) + yield + + +def _run_concurrently(fn, threads=6): + """Call fn from `threads` threads released at the same moment.""" + start = threading.Barrier(threads) + errors = [] + + def target(): + try: + start.wait(timeout=10) + fn() + except Exception as exc: # pragma: no cover - surfaced by the assert + errors.append(exc) + + workers = [threading.Thread(target=target) for _ in range(threads)] + for w in workers: + w.start() + for w in workers: + w.join(timeout=30) + assert not errors, errors + + +class _Counter: + """A constructor stand-in that is slow enough to overlap callers.""" + + def __init__(self): + self.calls = 0 + self._lock = threading.Lock() + + def __call__(self, *_args, **_kwargs): + with self._lock: + self.calls += 1 + time.sleep(0.05) + return object() + + +class _Settings: + """Stand-in for llama_index Settings, which type-checks embed_model.""" + + embed_model = None + + +def test_embed_model_loads_once(monkeypatch): + counter = _Counter() + monkeypatch.setattr(search, "FastEmbedEmbedding", counter) + monkeypatch.setattr(search, "_get_cached_model_path", lambda *_a, **_k: None) + monkeypatch.setattr(search, "Settings", _Settings) + config = { + "retrieval": {"embed_model_name": "m", "offline": False}, + "storage": {"model_cache_dir": "/nonexistent"}, + } + + _run_concurrently(lambda: search._ensure_embed_model(config)) + + assert counter.calls == 1 + assert search._embed_model_initialized is True + + +def test_reranker_loads_once(monkeypatch): + counter = _Counter() + monkeypatch.setattr("flashrank.Ranker", counter) + config = { + "retrieval": {"rerank_model_name": "ms-marco-MiniLM-L-12-v2", "offline": False}, + "storage": {"model_cache_dir": "/nonexistent"}, + } + + _run_concurrently(lambda: search._ensure_reranker(config)) + + assert counter.calls == 1 + + +def test_index_loads_once(monkeypatch, tmp_path): + counter = _Counter() + monkeypatch.setattr(search, "load_index_from_storage", counter) + monkeypatch.setattr(search.StorageContext, "from_defaults", lambda **_k: object()) + monkeypatch.setattr(search, "_ensure_embed_model", lambda _c: None) + config = {"storage": {"storage_dir": str(tmp_path)}} + + _run_concurrently(lambda: search.load_llamaindex_index(config)) + + assert counter.calls == 1 + + +def test_failed_reranker_load_is_not_cached(monkeypatch): + """A loader that raises must leave the cache empty so the next call retries. + + Double-checked locking is easy to get wrong here: assigning the global + before the constructor returns would publish a half-built object. + """ + attempts = [] + + def boom(*_args, **_kwargs): + attempts.append(1) + raise RuntimeError("no model in cache") + + monkeypatch.setattr("flashrank.Ranker", boom) + config = { + "retrieval": {"rerank_model_name": "ms-marco-MiniLM-L-12-v2", "offline": False}, + "storage": {"model_cache_dir": "/nonexistent"}, + } + + for _ in range(2): + with pytest.raises(RuntimeError): + search._ensure_reranker(config) + + assert len(attempts) == 2 + assert search._reranker_model is None + + +def test_warm_up_loads_everything(monkeypatch, tmp_path): + """warm_up must populate every cache a query needs.""" + loaded = [] + monkeypatch.setattr(search, "load_llamaindex_index", lambda _c: loaded.append("index")) + monkeypatch.setattr(search, "_ensure_embed_model", lambda _c: loaded.append("embed")) + monkeypatch.setattr(search, "_ensure_reranker", lambda _c: loaded.append("rerank")) + monkeypatch.setattr(search, "_ensure_bm25_retriever", lambda _c: loaded.append("bm25")) + monkeypatch.setattr(search, "get_heading_store", lambda: loaded.append("headings")) + config = { + "retrieval": {"offline": False}, + "storage": {"model_cache_dir": str(tmp_path), "storage_dir": str(tmp_path)}, + } + + search.warm_up(config) + + assert loaded == ["index", "embed", "rerank", "bm25", "headings"] + + +def test_warm_up_survives_a_failure(monkeypatch, tmp_path): + """A server that cannot preload is still worth starting.""" + def boom(*_a, **_k): + raise RuntimeError("storage missing") + + monkeypatch.setattr(search, "load_llamaindex_index", boom) + monkeypatch.setattr(search, "_ensure_embed_model", lambda _c: None) + monkeypatch.setattr(search, "_ensure_reranker", lambda _c: None) + monkeypatch.setattr(search, "_ensure_bm25_retriever", lambda _c: None) + monkeypatch.setattr(search, "get_heading_store", lambda: None) + config = { + "retrieval": {"offline": False}, + "storage": {"model_cache_dir": str(tmp_path), "storage_dir": str(tmp_path)}, + } + + search.warm_up(config) # must not raise From 3a8a745a3b46c7c9f5c28f0192e199c8dfc43f45 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:40:01 +0200 Subject: [PATCH 11/18] fix: hide internal error details from search callers run_search returned the raw exception text in its error field. That text can name filesystem paths and storage locations, which is not the caller's business - an MCP client gets a generic message and the operator gets the detail, with traceback, in the server log. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- src/chunksilo/search.py | 4 +++- test/test_error_handling.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/chunksilo/search.py b/src/chunksilo/search.py index 1a6101b..f26b4ed 100644 --- a/src/chunksilo/search.py +++ b/src/chunksilo/search.py @@ -1585,6 +1585,8 @@ def _mark(stage: str) -> None: return { "matched_files": [], "chunks": [], - "error": str(e), + # The exception text can name filesystem paths; the caller gets a + # generic message and the operator gets the detail in the log. + "error": "internal error during search (see server log)", "query": query, } diff --git a/test/test_error_handling.py b/test/test_error_handling.py index 6b56ac1..4e1563f 100644 --- a/test/test_error_handling.py +++ b/test/test_error_handling.py @@ -217,5 +217,21 @@ def test_state_creates_directory(self, tmp_path): assert isinstance(files, dict) +class TestSearchErrorResponses: + def test_internal_errors_are_not_leaked_to_the_caller(self, monkeypatch): + """Exception text can name filesystem paths; the caller gets a generic + message and the detail stays in the server log.""" + from chunksilo import search + + def boom(config): + raise FileNotFoundError("/srv/secret-storage/docstore.json is missing") + + monkeypatch.setattr(search, "load_llamaindex_index", boom) + result = search.run_search("anything") + + assert result["error"] == "internal error during search (see server log)" + assert "secret-storage" not in str(result) + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) From 364f14a7936e443db72c27012420335e7c188d00 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:44:56 +0200 Subject: [PATCH 12/18] feat: cap chunks per document in results and demote superseded revisions Search results were dominated by one document's chunks (or one document's many revisions): the reranker truncated to rerank_top_k and handed back whatever ranked highest, which for a long manual is five chunks of the same file. Results are now capped per document (retrieval.max_chunks_per_doc, default 2, 0 disables) with rank-order backfill, so the top-k spans documents instead of one document's every chunk; the reranker returns the full ranked list so the diversifier has candidates to backfill from. Chunks stamped superseded by the revision-aware indexer are demoted below current documents at both the candidate cap and the final cut, and BM25 file-name matches are deduped per document group with superseded revisions dropped. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- config.yaml | 1 + src/chunksilo/cfgload.py | 3 + src/chunksilo/search.py | 105 +++++++++++- test/test_search_diversity.py | 296 ++++++++++++++++++++++++++++++++++ 4 files changed, 398 insertions(+), 7 deletions(-) create mode 100644 test/test_search_diversity.py diff --git a/config.yaml b/config.yaml index 94ed2aa..8cd216e 100644 --- a/config.yaml +++ b/config.yaml @@ -99,6 +99,7 @@ retrieval: # score_threshold: 0.1 # recency_boost: 0.3 # recency_half_life_days: 365 + # max_chunks_per_doc: 2 # cap per document in the final results (0 disables) # Storage paths storage: diff --git a/src/chunksilo/cfgload.py b/src/chunksilo/cfgload.py index 23d5e76..35c216f 100644 --- a/src/chunksilo/cfgload.py +++ b/src/chunksilo/cfgload.py @@ -127,6 +127,9 @@ def _find_config() -> Path: "score_threshold": 0.1, "recency_boost": 0.3, "recency_half_life_days": 365, + # At most this many chunks of one document in the final results; + # freed slots go to the next-best other documents. 0 disables. + "max_chunks_per_doc": 2, "offline": False, }, "confluence": { diff --git a/src/chunksilo/search.py b/src/chunksilo/search.py index f26b4ed..7a8af5e 100644 --- a/src/chunksilo/search.py +++ b/src/chunksilo/search.py @@ -371,12 +371,25 @@ def _ensure_bm25_retriever(config: dict[str, Any]): def _format_bm25_matches(bm25_nodes: list[NodeWithScore], config: dict[str, Any]) -> list[dict[str, Any]]: - """Format BM25 file name matches for the response.""" + """Format BM25 file name matches for the response. + + Superseded revisions are dropped and each document group appears once + (best-scored member), so the short matched-files list is not spent on + many revisions of one document. + """ matched_files = [] + seen_groups: set[str] = set() for node in bm25_nodes: if node.score is None or node.score <= 0: continue + if _is_superseded(node): + continue metadata = node.node.metadata or {} + group = metadata.get("doc_group") + if group: + if group in seen_groups: + continue + seen_groups.add(group) file_path = metadata.get("file_path", "") source_uri, source_unc = ( _resolve_result_uris(file_path, config) if file_path else (None, None) @@ -1318,14 +1331,17 @@ def _retrieve_remote_sources( def _rerank_nodes( nodes: list[NodeWithScore], query: str, config: dict[str, Any] ) -> tuple[list[NodeWithScore], dict[int, float]]: - """Stage 2: Rerank nodes using FlashRank and return (nodes, scores).""" - rerank_top_k = config["retrieval"]["rerank_top_k"] + """Stage 2: Rerank nodes using FlashRank and return (nodes, scores). + + Returns the FULL candidate list in rerank order - truncation to + rerank_top_k happens after per-document diversification in run_search, + which needs the lower-ranked candidates to backfill from. + """ rerank_scores: dict[int, float] = {} if not nodes: return nodes, rerank_scores - rerank_limit = max(1, min(rerank_top_k, len(nodes))) try: reranker = _ensure_reranker(config) passages = [{"text": node.node.get_content() or ""} for node in nodes] @@ -1374,14 +1390,79 @@ def _rerank_nodes( reranked_nodes.append(node) rerank_scores[id(node)] = min_score - nodes = reranked_nodes[:rerank_limit] + nodes = reranked_nodes except Exception as e: logger.error(f"Reranking failed, falling back to vector search order: {e}") - nodes = nodes[:rerank_limit] return nodes, rerank_scores +def _is_superseded(node: NodeWithScore) -> bool: + return (node.node.metadata or {}).get("superseded") == "true" + + +def _partition_superseded(nodes: list[NodeWithScore]) -> list[NodeWithScore]: + """Stable partition: current documents first, superseded revisions after. + + Only relevant with indexing.versioning.index_superseded, where old + revisions stay in the index. They are demoted rather than dropped - being + findable when nothing current matches is the point of that mode. + """ + if not any(_is_superseded(node) for node in nodes): + return nodes + fresh = [n for n in nodes if not _is_superseded(n)] + old = [n for n in nodes if _is_superseded(n)] + return fresh + old + + +def _doc_key(node: NodeWithScore) -> str: + """Grouping key for diversification. + + doc_group covers revisions of one document; chunks indexed before that + metadata existed fall back to per-file grouping. Remote sources emit one + node per page or issue with a unique file_name, so they group per item. + """ + metadata = node.node.metadata or {} + return ( + metadata.get("doc_group") + or metadata.get("file_path") + or metadata.get("file_name") + or node.node.node_id + ) + + +def _diversify_by_group( + nodes: list[NodeWithScore], top_k: int, max_per_doc: int +) -> list[NodeWithScore]: + """Cap chunks per document in the final results, keeping rank order. + + Walks the ranked list taking at most max_per_doc chunks per document, + then backfills with the skipped chunks (still in rank order) so top_k + stays full when fewer documents match than top_k requires. Applied even + when everything fits in top_k: another document's best chunk belongs + above the fourth chunk of the same document. + """ + if max_per_doc <= 0: + return nodes[:top_k] + + taken: list[NodeWithScore] = [] + overflow: list[NodeWithScore] = [] + per_doc: dict[str, int] = {} + for node in nodes: + key = _doc_key(node) + if per_doc.get(key, 0) < max_per_doc: + per_doc[key] = per_doc.get(key, 0) + 1 + taken.append(node) + else: + overflow.append(node) + if len(taken) >= top_k: + break + + if len(taken) < top_k: + taken.extend(overflow[: top_k - len(taken)]) + return taken + + def _format_search_results( nodes: list[NodeWithScore], rerank_scores: dict[int, float], config: dict[str, Any] ) -> list[dict[str, Any]]: @@ -1532,7 +1613,9 @@ def _mark(stage: str) -> None: if recency_boost > 0: nodes = _apply_recency_boost(nodes, recency_boost, recency_half_life) - # Cap candidates before reranking + # Cap candidates before reranking; superseded revisions go to the + # back first so the cap trims them before current documents. + nodes = _partition_superseded(nodes) rerank_candidates = config["retrieval"]["rerank_candidates"] if len(nodes) > rerank_candidates: logger.info(f"Capping rerank candidates: {len(nodes)} -> {rerank_candidates}") @@ -1561,6 +1644,14 @@ def _mark(stage: str) -> None: ] logger.info(f"Phrase filter: {before_count} -> {len(nodes)} nodes for {required_phrases}") + # Vary the results across documents: demote superseded revisions, + # then cap chunks per document so one document (or its many + # revisions) cannot fill the whole top-k. + nodes = _partition_superseded(nodes) + rerank_top_k = config["retrieval"]["rerank_top_k"] + max_per_doc = config["retrieval"]["max_chunks_per_doc"] + nodes = _diversify_by_group(nodes, rerank_top_k, max_per_doc) + # Format results chunks = _format_search_results(nodes, rerank_scores, config) _mark("format") diff --git a/test/test_search_diversity.py b/test/test_search_diversity.py new file mode 100644 index 0000000..f562c30 --- /dev/null +++ b/test/test_search_diversity.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Tests for per-document result diversification and superseded demotion. + +One document's many chunks (or its many revisions) must not fill the whole +top-k; superseded revisions rank behind current documents. +""" + +import copy +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from llama_index.core import Settings +from llama_index.core.schema import NodeWithScore, TextNode + +from chunksilo import index, search +from chunksilo.index import build_index, DirectoryConfig, IndexConfig +from chunksilo.search import ( + _diversify_by_group, + _format_bm25_matches, + _partition_superseded, + run_search, +) + + +def _node(text, score=1.0, **metadata): + return NodeWithScore(node=TextNode(text=text, metadata=metadata), score=score) + + +# ============================================================================= +# Unit tests for the helpers +# ============================================================================= + + +class TestDiversifyByGroup: + def test_cap_enforced_and_next_group_pulled_in(self): + nodes = [ + _node("a1", doc_group="A"), + _node("a2", doc_group="A"), + _node("a3", doc_group="A"), + _node("b1", doc_group="B"), + ] + result = _diversify_by_group(nodes, top_k=3, max_per_doc=2) + assert [n.node.get_content() for n in result] == ["a1", "a2", "b1"] + + def test_backfill_keeps_top_k_full(self): + nodes = [_node(f"a{i}", doc_group="A") for i in range(6)] + result = _diversify_by_group(nodes, top_k=4, max_per_doc=2) + assert [n.node.get_content() for n in result] == ["a0", "a1", "a2", "a3"] + + def test_backfill_preserves_rank_order(self): + nodes = [ + _node("a1", doc_group="A"), + _node("a2", doc_group="A"), + _node("a3", doc_group="A"), + _node("b1", doc_group="B"), + ] + result = _diversify_by_group(nodes, top_k=4, max_per_doc=2) + assert [n.node.get_content() for n in result] == ["a1", "a2", "b1", "a3"] + + def test_file_path_fallback_for_unstamped_chunks(self): + nodes = [ + _node("x1", file_path="/s/x.txt"), + _node("x2", file_path="/s/x.txt"), + _node("y1", file_path="/s/y.txt"), + ] + result = _diversify_by_group(nodes, top_k=2, max_per_doc=1) + assert [n.node.get_content() for n in result] == ["x1", "y1"] + + def test_zero_disables_diversity_but_still_truncates(self): + nodes = [_node(f"a{i}", doc_group="A") for i in range(6)] + result = _diversify_by_group(nodes, top_k=5, max_per_doc=0) + assert len(result) == 5 + assert [n.node.get_content() for n in result] == ["a0", "a1", "a2", "a3", "a4"] + + +class TestPartitionSuperseded: + def test_stable_demotion(self): + nodes = [ + _node("old1", superseded="true"), + _node("fresh1"), + _node("old2", superseded="true"), + _node("fresh2"), + ] + result = _partition_superseded(nodes) + assert [n.node.get_content() for n in result] == [ + "fresh1", "fresh2", "old1", "old2", + ] + + def test_no_op_without_superseded(self): + nodes = [_node("a"), _node("b")] + assert _partition_superseded(nodes) is nodes + + +class TestBm25MatchedFiles: + def _config(self): + from chunksilo.cfgload import _DEFAULTS, _deep_merge + + return _deep_merge(_DEFAULTS, {}) + + def test_dedupe_by_group_and_drop_superseded(self): + nodes = [ + _node("m", 2.0, file_path="/s/manual_v2.txt", doc_group="g1"), + _node("m", 1.5, file_path="/s/manual_v1.txt", doc_group="g1"), + _node("m", 1.2, file_path="/s/old_A.txt", doc_group="g2", superseded="true"), + _node("m", 1.0, file_path="/s/other.txt", doc_group="g3"), + ] + matches = _format_bm25_matches(nodes, self._config()) + names = [Path(m["uri"]).name for m in matches] + assert names == ["manual_v2.txt", "other.txt"] + + def test_ungrouped_files_kept(self): + nodes = [ + _node("m", 2.0, file_path="/s/a.txt"), + _node("m", 1.0, file_path="/s/b.txt"), + ] + matches = _format_bm25_matches(nodes, self._config()) + assert len(matches) == 2 + + +# ============================================================================= +# End-to-end through run_search on a real mini index +# ============================================================================= + + +def _create_mock_embedding(): + from llama_index.core.embeddings import BaseEmbedding + + class MockEmbedding(BaseEmbedding): + def _get_vector(self, text: str): + vec = [0.0] * 384 + for word in text.lower().split(): + idx = sum(ord(c) for c in word) % 384 + vec[idx] += 1.0 + return vec + + def _get_query_embedding(self, query: str): + return self._get_vector(query) + + def _get_text_embedding(self, text: str): + return self._get_vector(text) + + async def _aget_query_embedding(self, query: str): + return self._get_vector(query) + + return MockEmbedding(model_name="mock-test") + + +class FakeRanker: + """Equal scores: rerank order collapses to retrieval order.""" + + def rerank(self, request): + return [{"id": i, "score": 0.9} for i in range(len(request.passages))] + + +T0, T1 = 1_600_000_000.0, 1_600_100_000.0 + + +def _search_config(storage_dir, model_dir): + from chunksilo.cfgload import _DEFAULTS, _deep_merge + + return _deep_merge( + _DEFAULTS, + { + "storage": {"storage_dir": str(storage_dir), "model_cache_dir": str(model_dir)}, + # embed_top_k above the corpus size so every chunk (including the + # small overview doc) reaches the diversifier deterministically. + "retrieval": {"recency_boost": 0.0, "embed_top_k": 100}, + }, + ) + + +def _build_corpus(base: Path) -> Path: + """Two revisions of a many-chunk manual plus one small overview doc.""" + data_dir = base / "data" + storage_dir = base / "storage" + data_dir.mkdir(parents=True, exist_ok=True) + + # ~7 chunks per revision at chunk_size 64: enough for the manual to crowd + # out the overview without a cap, while all chunks (2 revisions + the + # overview) still fit within embed_top_k so the diversifier sees them. + manual = " ".join( + f"widget assembly manual step {i}: attach the widget bracket and torque " + f"the widget bolts to spec." + for i in range(25) + ) + for name, content, mtime in [ + ("widget_manual_v1.txt", manual + " preliminary draft.", T0), + ("widget_manual_v2.txt", manual + " released edition.", T1), + ("widget_overview.txt", "widget overview: a short description of the widget product line.", T0), + ]: + path = data_dir / name + path.write_text(content) + os.utime(path, (mtime, mtime)) + + test_index_config = IndexConfig( + directories=[DirectoryConfig(path=data_dir)], chunk_size=64, chunk_overlap=8 + ) + orig = ( + index.STORAGE_DIR, + index.STATE_DB_PATH, + index.BM25_INDEX_DIR, + index.HEADING_STORE_PATH, + index._config, + ) + index.STORAGE_DIR = storage_dir + index.STATE_DB_PATH = storage_dir / "ingestion_state.db" + index.BM25_INDEX_DIR = storage_dir / "bm25_index" + index.HEADING_STORE_PATH = storage_dir / "heading_store.json" + index._heading_store = None + cfg = copy.deepcopy(index._config) + # Keep both revisions in the index so demotion is observable. + cfg.setdefault("indexing", {}).setdefault("versioning", {})["index_superseded"] = True + index._config = cfg + + mock_embed = _create_mock_embedding() + try: + with patch("chunksilo.index.load_index_config", return_value=test_index_config), \ + patch("chunksilo.index._create_fastembed_embedding", return_value=mock_embed), \ + patch("chunksilo.index.ensure_embedding_model_cached"), \ + patch("chunksilo.index.ensure_rerank_model_cached"): + build_index() + finally: + ( + index.STORAGE_DIR, + index.STATE_DB_PATH, + index.BM25_INDEX_DIR, + index.HEADING_STORE_PATH, + index._config, + ) = orig + index._heading_store = None + return storage_dir + + +@pytest.fixture(scope="module") +def corpus(tmp_path_factory): + base = tmp_path_factory.mktemp("diversity_corpus") + storage_dir = _build_corpus(base) + return base, storage_dir + + +@pytest.fixture +def search_env(corpus, monkeypatch): + base, storage_dir = corpus + config = _search_config(storage_dir, base / "models") + + monkeypatch.setattr(search, "_config", config) + monkeypatch.setattr(search, "_index_cache", None) + monkeypatch.setattr(search, "_bm25_retriever_cache", None) + monkeypatch.setattr(search, "_embed_model_initialized", True) + monkeypatch.setattr(search, "_reranker_model", FakeRanker()) + monkeypatch.setattr(index, "HEADING_STORE_PATH", storage_dir / "heading_store.json") + monkeypatch.setattr(index, "_heading_store", None) + Settings.embed_model = _create_mock_embedding() + yield config + + +def result_names(result) -> list[str]: + assert "error" not in result, result.get("error") + return [Path(c["location"]["uri"] or "").name for c in result["chunks"]] + + +QUERY = "widget assembly manual" + + +class TestSearchDiversity: + def test_results_vary_across_documents(self, search_env): + names = result_names(run_search(QUERY)) + # The manual revisions share one doc_group: at most 2 of its chunks, + # and the small overview document still makes the cut. + assert "widget_overview.txt" in names + assert 0 < len(names) <= search_env["retrieval"]["rerank_top_k"] + + def test_superseded_revision_never_shown_over_current(self, search_env): + names = result_names(run_search(QUERY)) + assert "widget_manual_v1.txt" not in names + assert "widget_manual_v2.txt" in names + + def test_cap_disabled_restores_flat_ranking(self, search_env, monkeypatch): + config = copy.deepcopy(search_env) + config["retrieval"]["max_chunks_per_doc"] = 0 + monkeypatch.setattr(search, "_config", config) + names = result_names(run_search(QUERY)) + # Without the cap the many manual chunks crowd out the overview. + assert "widget_overview.txt" not in names + assert len(names) == config["retrieval"]["rerank_top_k"] + + def test_bm25_matched_files_deduped_per_group(self, search_env): + result = run_search("widget manual") + matched = [Path(f["uri"]).name for f in result["matched_files"]] + # One entry for the manual group (the current revision), not one per + # revision file. + assert matched.count("widget_manual_v2.txt") <= 1 + assert "widget_manual_v1.txt" not in matched From c2304008b790e9af3fddcd4a900be3021d12190e Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:50:23 +0200 Subject: [PATCH 13/18] feat: add an unauthenticated streamable-http transport chunksilo-mcp can now serve MCP over streamable-http as well as stdio: server.transport selects it (or --transport overrides the config), and server.host / server.port give the bind address, loopback by default. On the HTTP path the server warms up the index, the models and the BM25 index before accepting the first request, so concurrent first calls do not all wait on the slowest cold start. The transport performs no authentication of its own. That is deliberate: login, TLS termination and access policy belong to a reverse proxy in front of the server, which is why the default bind is 127.0.0.1 and the log line at startup says so. The stdio server is unchanged apart from its MCP name, which is now "chunksilo". Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- config.yaml | 9 +++++ src/chunksilo/cfgload.py | 8 ++++ src/chunksilo/server.py | 73 ++++++++++++++++++++++++++++++++---- test/test_server.py | 81 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 8 deletions(-) diff --git a/config.yaml b/config.yaml index 8cd216e..c3f0a60 100644 --- a/config.yaml +++ b/config.yaml @@ -106,6 +106,15 @@ storage: storage_dir: "./storage" model_cache_dir: "./models" +# MCP transport. stdio (the default) is started by the MCP client itself. +# streamable-http listens on host:port for remote clients. It performs no +# authentication of its own, so keep the loopback bind unless an +# authenticating reverse proxy (TLS termination, login) sits in front of it. +# server: +# transport: "stdio" # stdio | streamable-http +# host: "127.0.0.1" +# port: 8400 + # Network shares (optional): present files indexed from a mounted share at the # share's own location - an smb:// URI plus the Windows UNC path - instead of # the local mount path, so clients on other machines get a location they can diff --git a/src/chunksilo/cfgload.py b/src/chunksilo/cfgload.py index 35c216f..ec2c465 100644 --- a/src/chunksilo/cfgload.py +++ b/src/chunksilo/cfgload.py @@ -158,6 +158,14 @@ def _find_config() -> Path: "storage_dir": "./storage", "model_cache_dir": "./models", }, + "server": { + "transport": "stdio", # stdio | streamable-http + # Loopback by default. The HTTP transport performs no authentication + # of its own: exposing the port is a deliberate act, taken together + # with TLS termination and a login layer in front of this server. + "host": "127.0.0.1", + "port": 8400, + }, # Present files indexed from a mounted network share at the share's own # location (an smb:// URI plus the Windows UNC path) instead of the # server-local mount path, so a client on another machine gets a location diff --git a/src/chunksilo/server.py b/src/chunksilo/server.py index 3a20210..adb47fb 100644 --- a/src/chunksilo/server.py +++ b/src/chunksilo/server.py @@ -3,6 +3,14 @@ """ MCP server for querying documentation using RAG. Returns raw document chunks for the calling LLM to synthesize. + +Two transports: +- stdio (default): the MCP client starts this process itself; one user, one + machine. +- streamable-http: listens on server.host:server.port for any MCP client that + can reach it. This server performs NO authentication of its own - keep the + loopback bind (the default) or put an authenticating reverse proxy in front + of it before exposing the port. """ import argparse import asyncio @@ -22,6 +30,8 @@ LOG_MAX_SIZE_BYTES = 10 * 1024 * 1024 LOG_BACKUP_COUNT = 5 +TRANSPORTS = ("stdio", "streamable-http") + # Module-level state (set during initialization) _server_config_path: Path | None = None _mcp: FastMCP | None = None @@ -59,11 +69,29 @@ def _setup_logging(storage_dir: str | Path): logging.getLogger("llama_index.readers.confluence").setLevel(logging.WARNING) -def _create_server() -> FastMCP: - """Create and configure the MCP server with tools.""" +def _create_server( + config: dict[str, Any] | None = None, transport: str = "stdio" +) -> FastMCP: + """Create and configure the MCP server with tools. + + For ``streamable-http`` the bind address comes from ``config["server"]``; + the stdio server needs no configuration at all. + """ from .search import run_search - mcp = FastMCP("llamaindex-docs-rag") + if transport == "streamable-http": + server_cfg = config["server"] + mcp = FastMCP( + "chunksilo", + host=server_cfg["host"], + port=int(server_cfg["port"]), + # Each POST is independent - no session affinity - so a tool call + # never depends on which worker served the previous one. + stateless_http=True, + json_response=True, + ) + else: + mcp = FastMCP("chunksilo") @mcp.tool() async def search_docs( @@ -80,7 +108,7 @@ async def search_docs( return mcp -def run_server(config_path: Path | None = None): +def run_server(config_path: Path | None = None, transport_override: str | None = None): """Start the MCP server.""" global _server_config_path, _mcp @@ -98,9 +126,33 @@ def run_server(config_path: Path | None = None): # before the config was read. _setup_logging(config["storage"]["storage_dir"]) logger.info("Starting ChunkSilo MCP server") + transport = transport_override or config["server"]["transport"] + + if transport == "stdio": + _mcp = _create_server(config, transport) + _mcp.run() + return - _mcp = _create_server() - _mcp.run() + if transport != "streamable-http": + raise SystemExit( + f"Unknown server.transport: {transport!r} (expected one of {', '.join(TRANSPORTS)})" + ) + + # Pay for the index, the models and the BM25 index once, before the first + # request - concurrent first requests would otherwise all wait on the + # slowest cold start. + from . import search as search_module + + logger.info("Warming up the search pipeline before accepting requests") + search_module.warm_up(config) + + _mcp = _create_server(config, transport) + logger.info( + "Serving MCP over streamable-http on %s:%s (no authentication - front it with a proxy before exposing it)", + config["server"]["host"], + config["server"]["port"], + ) + _mcp.run(transport="streamable-http") def main(): @@ -109,13 +161,18 @@ def main(): parser = argparse.ArgumentParser( prog="chunksilo-mcp", - description="Run ChunkSilo MCP server (stdio transport)", + description="Run ChunkSilo MCP server (stdio or streamable-http transport)", ) parser.add_argument("--config", help="Path to config.yaml") + parser.add_argument( + "--transport", + choices=list(TRANSPORTS), + help="Override server.transport from config", + ) args = parser.parse_args() config_path = Path(args.config) if args.config else None - run_server(config_path) + run_server(config_path, transport_override=args.transport) if __name__ == "__main__": diff --git a/test/test_server.py b/test/test_server.py index 2e0a5f2..6088e08 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -4,6 +4,8 @@ import asyncio from unittest.mock import patch +import pytest + from chunksilo.server import _create_server, _setup_logging # ============================================================================= @@ -130,3 +132,82 @@ def test_main_no_config(self, mock_logging, mock_run_server): mock_run_server.assert_called_once() config_arg = mock_run_server.call_args[0][0] assert config_arg is None + + @patch("chunksilo.server.run_server") + def test_main_passes_transport_override(self, mock_run_server): + from chunksilo.server import main + + with patch("sys.argv", ["chunksilo-mcp", "--transport", "streamable-http"]): + main() + + assert mock_run_server.call_args.kwargs["transport_override"] == "streamable-http" + + def test_main_rejects_unknown_transport(self): + from chunksilo.server import main + + with patch("sys.argv", ["chunksilo-mcp", "--transport", "carrier-pigeon"]): + with pytest.raises(SystemExit): + main() + + +# ============================================================================= +# Tests for the transports +# ============================================================================= + + +class TestTransports: + HTTP_CONFIG = { + "server": {"transport": "streamable-http", "host": "127.0.0.1", "port": 8765}, + } + + def test_http_server_binds_the_configured_address(self): + mcp = _create_server(self.HTTP_CONFIG, "streamable-http") + + assert mcp.settings.host == "127.0.0.1" + assert mcp.settings.port == 8765 + assert mcp.settings.stateless_http is True + assert mcp.settings.json_response is True + tool_names = [t.name for t in mcp._tool_manager.list_tools()] + assert "search_docs" in tool_names + + def _run(self, config, monkeypatch, tmp_path, transport_override=None): + """Run run_server with the network, logging and warm-up stubbed out.""" + from chunksilo import search + from chunksilo import server as server_module + + config = {**config, "storage": {"storage_dir": str(tmp_path)}} + calls = {"warm_up": [], "run": []} + monkeypatch.setattr("chunksilo.cfgload.load_config", lambda *_a, **_k: config) + monkeypatch.setattr(server_module, "_setup_logging", lambda _d: None) + monkeypatch.setattr(search, "warm_up", lambda cfg: calls["warm_up"].append(cfg)) + monkeypatch.setattr( + server_module.FastMCP, "run", lambda self, **kw: calls["run"].append(kw) + ) + server_module.run_server(None, transport_override=transport_override) + return calls + + def test_http_transport_warms_up_then_serves(self, monkeypatch, tmp_path): + calls = self._run(self.HTTP_CONFIG, monkeypatch, tmp_path) + + assert len(calls["warm_up"]) == 1 + assert calls["run"] == [{"transport": "streamable-http"}] + + def test_stdio_transport_stays_lazy(self, monkeypatch, tmp_path): + config = {"server": {"transport": "stdio", "host": "127.0.0.1", "port": 8400}} + + calls = self._run(config, monkeypatch, tmp_path) + + assert calls["warm_up"] == [] + assert calls["run"] == [{}] + + def test_override_wins_over_config(self, monkeypatch, tmp_path): + calls = self._run(self.HTTP_CONFIG, monkeypatch, tmp_path, transport_override="stdio") + + assert calls["warm_up"] == [] + assert calls["run"] == [{}] + + def test_unknown_transport_exits(self, monkeypatch, tmp_path): + config = {"server": {"transport": "carrier-pigeon", "host": "127.0.0.1", "port": 1}} + + with pytest.raises(SystemExit): + self._run(config, monkeypatch, tmp_path) From eee207ea23b80ed5746c73e309bea8c8454c86d7 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 19:59:01 +0200 Subject: [PATCH 14/18] docs: split the README into docs/ and describe the new indexing, search and server behaviour The README grew into a 400-line manual. It is now a short overview - pitch, installation, a config sketch, the search_docs result shape and a documentation index - with the reference material in docs/: the full configuration reference (including the flattened indexing options, revision handling, share locations, the HTTP transport and an upgrade table for 2.x configs), CLI usage with --check-files, the tool result fields, MCP client setup for Claude Code, Claude Desktop, Cline, Roo Code and opencode over stdio or HTTP, and troubleshooting notes for the scan-health, revision and re-indexing behaviour. AGENTS.md records the invariants the code now relies on: the config activation contract, indexing never writing inside an indexed directory, withheld deletions, one state row per attempted file, the dry-run scan reusing the indexer's own scan, filesystem-free result formatting, the locked lazy loaders, and the HTTP transport authenticating nobody. The release bundle now ships docs/, and the test docs no longer point at a test/requirements.txt that does not exist. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- AGENTS.md | 23 ++- README.md | 349 ++++++-------------------------------- docs/cli.md | 79 +++++++++ docs/configuration.md | 267 +++++++++++++++++++++++++++++ docs/mcp-clients.md | 188 ++++++++++++++++++++ docs/tools.md | 77 +++++++++ docs/troubleshooting.md | 19 +++ scripts/prepare-common.sh | 1 + test/README.md | 6 +- test/TESTING.md | 4 +- test/run_tests.sh | 4 +- 11 files changed, 708 insertions(+), 309 deletions(-) create mode 100644 docs/cli.md create mode 100644 docs/configuration.md create mode 100644 docs/mcp-clients.md create mode 100644 docs/tools.md create mode 100644 docs/troubleshooting.md diff --git a/AGENTS.md b/AGENTS.md index 8404283..19d960f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,8 +8,9 @@ **Before committing any changes, run the test suite:** ```bash -python3 -m pytest test/ -v --ignore=test/test_rag_metrics.py +test/run_tests.sh ``` +(equivalent to `python3 -m pytest test/ -v --ignore=test/test_rag_metrics.py --ignore=test/test_indexing_benchmark.py`), and `ruff check src/`. **Requirements:** - Python 3.11 is required for running tests @@ -23,3 +24,23 @@ python3 -m pytest test/ -v --ignore=test/test_rag_metrics.py ## Documentation Guidelines - **Do not claim absolute data privacy**: ChunkSilo runs search locally, but results are passed to the user's MCP client LLM, which may be cloud-hosted. Never state or imply that "no data leaves the network" — only state that ChunkSilo itself does not make external calls (when offline mode is enabled). The distinction is: ChunkSilo doesn't phone home, but the LLM client receiving results may. +- The README is a short overview; reference material lives in `docs/` (one file per topic, each starting with `Part of [ChunkSilo](../README.md).`). Keep `config.yaml`, `chunksilo --dump-defaults` and `docs/configuration.md` in agreement. + +## Configuration Invariants +- **Config activation contract**: `index.py` reads config at import time via `load_config()`, and an explicit-path `load_config(path)` becomes the process-wide active config that `cfgload.get()` answers from. `cli.main`, `run_server` and `build_index` therefore call `load_config(path)` before importing/using `.search`/`.index`; keep that ordering if refactoring startup. Tests get a fresh active config per test from the autouse fixture in `test/conftest.py`. +- **Removed options warn, they do not silently vanish**: when an option is dropped from the schema, add it to `cfgload._REMOVED_KEYS` with its replacement. + +## Indexing Invariants +- **Indexing never writes inside an indexed directory.** All scratch space (the + .doc conversion work dirs, the LibreOffice profile), the index, and the logs + live under `storage.storage_dir`; sources are opened read-only and .doc files + are copied out before LibreOffice ever sees them. `test/test_never_mutates.py` + enforces this with a byte-for-byte snapshot - keep it passing. +- **A file missing from a scan is only deleted if the scan could see where it lives.** `LocalFileSystemSource` records unreadable paths and incomplete walks on `ScanStatus`; `build_index` subtracts those from the deletion set. Pruning on a failed lookup drops the file's chunks and state row, so the next run re-indexes it — on a laggy network mount that becomes a permanent re-index loop. +- **Every attempted file gets a state row unless it failed.** `load_files_parallel` returns a `LoadOutcome` per file: `empty` (extracted no text) is a final answer and is persisted with no doc ids; `failed` gets no row so it is retried. Dropping empty results makes those files look new on every run for ever. +- **`--check-files` runs the indexer's own scan.** `filecheck.run_check` drives the same `iter_candidates` / `_match_decision` / revision partitioning as `build_index`; a change to the scan that is not visible through those APIs will make the dry run lie. + +## Search Invariants +- **Result formatting must not touch the filesystem.** Result paths often live on a network mount where a stat plus a readlink per component dominates the whole query; `_normalize_result_path` and the share-location mapping (`shareuri.py`) work from the stored string and the configuration alone. Resolving would also rewrite a path into the mount's internal form, which no application can open. `test/test_chunk_location.py` pins this. +- **The lazy loaders are double-checked-locked.** `_ensure_embed_model`, `_ensure_reranker`, `_ensure_bm25_retriever` and `load_llamaindex_index` are process-wide; over the HTTP transport tool calls run concurrently in a thread pool with no session affinity, so without the locks each first caller cold-starts its own copy. `run_server` calls `search.warm_up` before serving HTTP. +- **The HTTP transport authenticates nobody.** Access control belongs to a reverse proxy in front of the server; do not add credentials, tokens or per-user filtering to the transport itself. diff --git a/README.md b/README.md index 4778215..e48d444 100644 --- a/README.md +++ b/README.md @@ -12,37 +12,14 @@ ChunkSilo is like a local Google for your documents. It uses semantic search — - Runs entirely on your machine — no servers, no infrastructure - Semantic search + keyword filename matching across PDF, DOCX, DOC, Markdown, and TXT -- Incremental indexing — only reprocesses new or changed files -- Heading-aware results with source links back to the original file +- Incremental indexing — only reprocesses new or changed files, and never writes inside an indexed directory +- Revision-aware indexing — files that are revisions of one document (`Spec_v2.pdf`, `.../Rev B/Spec.docx`) are grouped and only the newest is indexed; review-comment copies are skipped +- Heading-aware results with source links back to the original file, spread across documents instead of one document's every chunk - Date filtering and recency boosting +- Files indexed from a mounted network share can be presented at the share's own location (an `smb://` URI plus the Windows UNC path) +- Serves MCP over stdio to a local client, or over streamable-http behind a reverse proxy of your own - Optional Confluence and Jira integrations (supports Cloud and Server/Data Center) -### Example `search_docs` output - -```json -{ - "matched_files": [ - { "uri": "file:///docs/database-configuration.docx", "score": 0.8432 } - ], - "num_matched_files": 1, - "chunks": [ - { - "text": "To configure the database connection, set the DATABASE_URL environment variable...", - "score": 0.912, - "location": { - "uri": "file:///docs/setup-guide.pdf", - "page": 12, - "line": null, - "heading_path": ["Getting Started", "Configuration", "Database"] - } - } - ], - "num_chunks": 1, - "query": "how to configure the database", - "retrieval_time": "0.42s" -} -``` - ## Installation ### Option A: Install from PyPI (Recommended) @@ -60,7 +37,7 @@ for backward compatibility.) Then: 1. **Create** a config file at `~/.config/chunksilo/config.yaml` (see [Configuration](#configuration)) 2. **Build** the index: `chunksilo --build-index` -3. **Configure** your MCP client (see [MCP Client Configuration](#mcp-client-configuration)) +3. **Configure** your MCP client (see [docs/mcp-clients.md](docs/mcp-clients.md)) ### Option B: Offline Bundle @@ -79,306 +56,76 @@ cd chunksilo 3. **Edit** `config.yaml` to set your document directories 4. **Build** the index: `./venv/bin/chunksilo --build-index` -5. **Configure** your MCP client (see [MCP Client Configuration](#mcp-client-configuration)) +5. **Configure** your MCP client (see [docs/mcp-clients.md](docs/mcp-clients.md)) ## Configuration -ChunkSilo uses a single configuration file: `config.yaml` - -### Configuration File - -Edit `config.yaml` to configure your settings: +ChunkSilo reads one file, `config.yaml`, taken from `--config`, else the +`CHUNKSILO_CONFIG` environment variable, else `./config.yaml`, else +`~/.config/chunksilo/config.yaml`. Every setting is optional and +`chunksilo --dump-defaults` prints them all with their defaults. ```yaml -# Indexing settings - used by chunksilo --build-index indexing: directories: - "./data" - - "/mnt/nfs/shared-docs" - - path: "/mnt/samba/engineering" - include: ["**/*.pdf", "**/*.md"] - exclude: ["**/archive/**"] - chunk_size: 1600 - chunk_overlap: 200 + - "/mnt/docs" -# Retrieval settings - used when searching retrieval: - embed_top_k: 20 rerank_top_k: 5 - score_threshold: 0.1 -# Confluence integration (optional) - supports Cloud and Server/Data Center -confluence: - url: "https://confluence.example.com" - username: "your-username" - api_token: "your-api-token" - -# Storage paths (usually don't need to change) storage: storage_dir: "./storage" model_cache_dir: "./models" ``` -All settings are optional and have sensible defaults. - -### Configuration Reference - -> **Tip:** Run `chunksilo --dump-defaults` to see all available options with their default values. - -#### Indexing Settings - -| Setting | Description | -| :--- | :--- | -| `indexing.directories` | List of directories to index (strings or objects) | -| `indexing.chunk_size` | Maximum size of text chunks | -| `indexing.chunk_overlap` | Overlap between adjacent chunks | - -**Per-directory options** (when using object format): - -| Option | Description | -| :--- | :--- | -| `path` | Directory path to index (required) | -| `include` | Glob patterns for files to include | -| `exclude` | Glob patterns for files to exclude | -| `recursive` | Whether to recurse into subdirectories | -| `enabled` | Whether to index this directory | - -**Project-wide directory defaults** — set once instead of repeating per directory: - -| Option | Description | -| :--- | :--- | -| `indexing.defaults.include` | Default include patterns for all directories | -| `indexing.defaults.exclude` | Default exclude patterns for all directories | -| `indexing.defaults.recursive` | Default recursive setting for all directories | - -**Advanced indexing options** — performance tuning, timeouts, and diagnostics: - -| Setting | Description | -| :--- | :--- | -| `indexing.parallel_workers` | Threads for parallel file loading (1 = serial) | -| `indexing.batch_size` | Upper bound on the embedding batch size; sizing adapts down under memory pressure | -| `indexing.checkpoint_interval_files` | Files processed between index checkpoints | -| `indexing.checkpoint_interval_seconds` | Seconds between index checkpoints | -| `indexing.per_file_seconds` | Give up on a single file after this many seconds (0 disables) | -| `indexing.scan_item_seconds` | Timeout for stat/hash/walk operations while scanning | -| `indexing.slow_file_threshold_seconds` | Warn when one file takes longer than this | - -#### Retrieval Settings - -| Setting | Description | -| :--- | :--- | -| `retrieval.embed_model_name` | Embedding model for vector search | -| `retrieval.embed_top_k` | Candidates from vector search before reranking | -| `retrieval.rerank_model_name` | Reranker model | -| `retrieval.rerank_top_k` | Final results after reranking | -| `retrieval.rerank_candidates` | Maximum candidates sent to reranker | -| `retrieval.score_threshold` | Minimum score (0.0-1.0) for results | -| `retrieval.recency_boost` | Recency boost weight (0.0-1.0) | -| `retrieval.recency_half_life_days` | Days until recency boost halves | - -| `retrieval.offline` | Prevent ML library network requests | - -#### Confluence Settings (optional) - -> **Note:** Confluence support is installed by default; just set the values below to enable it. - -| Setting | Description | -| :--- | :--- | -| `confluence.url` | Confluence base URL (empty = disabled) | -| `confluence.username` | Confluence username | -| `confluence.api_token` | Confluence API token (Cloud) or Personal Access Token (Server/Data Center) | -| `confluence.timeout` | Request timeout in seconds | -| `confluence.max_results` | Maximum results per search | - -**Creating a Confluence API Token:** -1. Log into Confluence -2. Go to Account Settings > Security > API Tokens (for Cloud) or User Profile > Personal Access Tokens (for Server/Data Center) -3. Click "Create API Token" or "Create Token" -4. Copy the token and add it to your config - -#### Jira Settings (optional) - -> **Note:** Jira support is installed by default; just set the values below to enable it. - -| Setting | Description | -| :--- | :--- | -| `jira.url` | Jira base URL (empty = disabled) | -| `jira.username` | Jira username/email | -| `jira.api_token` | Jira API token | -| `jira.timeout` | Request timeout in seconds | -| `jira.max_results` | Maximum results per search | -| `jira.projects` | Project keys to search (empty = all) | -| `jira.include_comments` | Include issue comments in search | -| `jira.include_custom_fields` | Include custom fields in search | - -**Creating a Jira API Token:** -1. Log into Jira -2. Go to Account Settings > Security > API Tokens -3. Click "Create API Token" -4. Copy the token and add it to your config - -#### SSL Settings (optional) - -| Setting | Description | -| :--- | :--- | -| `ssl.ca_bundle_path` | Path to custom CA bundle file | - -#### Storage Settings - -| Setting | Description | -| :--- | :--- | -| `storage.storage_dir` | Directory for vector index and state | -| `storage.model_cache_dir` | Directory for model cache | - -## CLI Usage - -```bash -chunksilo --build-index # Build or update the search index -chunksilo "your search query" # Search for documents -chunksilo "report" --date-from 2024-01-01 --date-to 2024-03-31 # Date filtering -chunksilo --dump-defaults # Print all config options with defaults -``` - -### CLI Options - -| Option | Description | -| :--- | :--- | -| `query` | Search query text (positional argument) | -| `--build-index` | Build or update the search index with step-by-step progress output, then exit | -| `--download-models` | Download required ML models, then exit | -| `--dump-defaults` | Print all default configuration values as YAML, then exit | -| `--date-from` | Start date filter (YYYY-MM-DD format, inclusive) | -| `--date-to` | End date filter (YYYY-MM-DD format, inclusive) | -| `--json` | Output results as JSON instead of formatted text | -| `-v, --verbose` | Show diagnostic messages (model loading, search stats) | -| `--config` | Path to config.yaml (overrides auto-discovery) | -| `CHUNKSILO_CONFIG` | Environment variable alternative to `--config` | - -## MCP Client Configuration +The full reference — per-directory filters, revision handling, retrieval +tuning, Confluence and Jira, share locations, the HTTP transport, and what +changed for users upgrading from 2.x — is in +[docs/configuration.md](docs/configuration.md). -Configure your MCP client to run ChunkSilo. Below are examples for common clients. +## The `search_docs` tool -> **Note:** For PyPI installs, use `chunksilo-mcp` directly. For offline bundles, use the full path `/path/to/chunksilo/venv/bin/chunksilo-mcp`. You can find the PyPI-installed binary location with `which chunksilo-mcp`. +`search_docs` answers with the files whose names matched and the individual +chunks that carry the answer, each located by a URI, the page or line it came +from, and its heading path: -### Claude Code - -Add chunksilo as an MCP server using the CLI: - -**PyPI install:** -```bash -claude mcp add chunksilo --scope user -- chunksilo-mcp --config ~/.config/chunksilo/config.yaml -``` - -**Offline bundle:** -```bash -claude mcp add chunksilo --scope user -- /path/to/chunksilo/venv/bin/chunksilo-mcp --config /path/to/chunksilo/config.yaml -``` - -Verify it's connected: - -```bash -claude mcp list -``` - -### Claude Desktop - -Add to `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows): - -**PyPI install:** -```json -{ - "mcpServers": { - "chunksilo": { - "command": "chunksilo-mcp", - "args": ["--config", "/path/to/config.yaml"] - } - } -} -``` - -**Offline bundle:** -```json -{ - "mcpServers": { - "chunksilo": { - "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp", - "args": ["--config", "/path/to/chunksilo/config.yaml"] - } - } -} -``` - -### Cline (VS Code Extension) - -Add to `cline_mcp_settings.json` (typically in `~/.config/Code/User/globalStorage/saoudrizwan.claude-dev/settings/`): - -**PyPI install:** -```json -{ - "mcpServers": { - "chunksilo": { - "command": "chunksilo-mcp", - "args": ["--config", "/path/to/config.yaml"], - "disabled": false, - "autoApprove": [] - } - } -} -``` - -**Offline bundle:** ```json { - "mcpServers": { - "chunksilo": { - "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp", - "args": ["--config", "/path/to/chunksilo/config.yaml"], - "disabled": false, - "autoApprove": [] - } - } -} -``` - -### Roo Code (VS Code Extension) - -Add to `mcp_settings.json` (typically in `~/.config/Code/User/globalStorage/rooveterinaryinc.roo-cline/settings/`): - -**PyPI install:** -```json -{ - "mcpServers": { - "chunksilo": { - "command": "chunksilo-mcp", - "args": ["--config", "/path/to/config.yaml"] + "matched_files": [ + { "uri": "file:///docs/database-configuration.docx", "unc": null, "score": 0.8432 } + ], + "num_matched_files": 1, + "chunks": [ + { + "text": "To configure the database connection, set the DATABASE_URL environment variable...", + "score": 0.912, + "location": { + "uri": "file:///docs/setup-guide.pdf", + "unc": null, + "page": 12, + "line": null, + "heading_path": ["Getting Started", "Configuration", "Database"] + } } - } + ], + "num_chunks": 1, + "query": "how to configure the database", + "retrieval_time": "0.42s" } ``` -**Offline bundle:** -```json -{ - "mcpServers": { - "chunksilo": { - "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp", - "args": ["--config", "/path/to/chunksilo/config.yaml"] - } - } -} -``` +See [docs/tools.md](docs/tools.md) for every field, including how files on a +mounted network share are presented. -## Troubleshooting +## Documentation -- **Index missing**: Run `chunksilo --build-index` (PyPI install) or `./venv/bin/chunksilo --build-index` (offline bundle). -- **Retrieval errors**: Check paths in your MCP client configuration. -- **Offline mode**: PyPI installs default to `offline: false` (models auto-download). The offline bundle includes pre-downloaded models and sets `offline: true`. Set `retrieval.offline: true` in `config.yaml` to prevent network calls after initial model download. -- **Confluence Integration**: Included by default — set `confluence.url`, `confluence.username`, and `confluence.api_token` in `config.yaml`. -- **Jira Integration**: Included by default — set `jira.url`, `jira.username`, and `jira.api_token` in `config.yaml`. Optionally configure `jira.projects` to restrict search to specific project keys. -- **Custom CA Bundle**: Set `ssl.ca_bundle_path` in `config.yaml` for custom certificates. -- **Network mounts**: Unavailable directories are skipped with a warning; indexing continues with available directories. -- **Legacy .doc files**: Requires LibreOffice to be installed for automatic conversion to .docx. If LibreOffice is not found, .doc files are skipped with a warning. Full heading extraction is supported. +- [docs/configuration.md](docs/configuration.md) — the config file, where it is discovered, and a reference for every setting. +- [docs/cli.md](docs/cli.md) — `chunksilo` and `chunksilo-mcp` options, including `--check-files` for explaining why a file is or is not indexed. +- [docs/tools.md](docs/tools.md) — the `search_docs` result shape. +- [docs/mcp-clients.md](docs/mcp-clients.md) — client setup for Claude Code, Claude Desktop, Cline, Roo Code and opencode, over stdio or HTTP. +- [docs/troubleshooting.md](docs/troubleshooting.md) — indexing surprises, network mounts, offline mode and more. ## License -Apache-2.0. See [LICENSE](LICENSE) for details. +Apache-2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE). diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..db34303 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,79 @@ +# CLI Usage + +Part of [ChunkSilo](../README.md). + +```bash +chunksilo --build-index # Build or update the search index +chunksilo "your search query" # Search for documents +chunksilo "report" --date-from 2024-01-01 --date-to 2024-03-31 # Date filtering +chunksilo --dump-defaults # Print all config options with defaults +chunksilo --list-files # List every indexed file +chunksilo --check-files # Dry-run the file filters, report every verdict +chunksilo --check-files /path/to/file # Explain why one file is or is not indexed +``` + +## CLI Options + +| Option | Description | +| :--- | :--- | +| `query` | Search query text (positional argument) | +| `--build-index` | Build or update the search index with step-by-step progress output, then exit | +| `--download-models` | Download required ML models, then exit | +| `--dump-defaults` | Print all default configuration values as YAML, then exit | +| `--date-from` | Start date filter (YYYY-MM-DD format, inclusive) | +| `--date-to` | End date filter (YYYY-MM-DD format, inclusive) | +| `--json` | Output results as JSON instead of formatted text | +| `-v, --verbose` | Show diagnostic messages (model loading, search stats, every file the indexer skipped or reprocessed and why) | +| `--list-files` | List all indexed file paths, then exit | +| `--check-files [PATH]` | Dry-run the indexing file filters without building the index, then exit. Reports a verdict per file; with PATH, explains that one file (see below) | +| `--config` | Path to config.yaml (overrides auto-discovery) | +| `CHUNKSILO_CONFIG` | Environment variable alternative to `--config` | + +## Testing the file filters (`--check-files`) + +`--check-files` runs the exact scan the indexer uses — directory availability, +include/exclude globs, directory pruning, revision grouping, review-copy +detection — and reports what would happen, without touching the index, the +state database, or the models. Use it to answer "why is this file (not) being +indexed?" without re-indexing: + +```text +$ chunksilo --check-files +State DB: ./storage/ingestion_state.db (present) + +/srv/docs + scanned (recursive) + pruned dir /srv/docs/.git [by '**/.git/**'] + superseded /srv/docs/Spec A.docx [older revision; latest in group is /srv/docs/Spec B.docx] + would index /srv/docs/Spec B.docx [matched include pattern '**/*.docx'] (indexed) + review copy /srv/docs/Spec review.docx [review copy (filename matched '\breview\b'); never indexed] + not included /srv/docs/notes.xlsx [no include pattern matched] + +Summary: 1 would index, 1 superseded, 1 review copies, 1 not matching any include, 1 directories pruned +``` + +With a path argument it explains that one file, including its revision group: + +```text +$ chunksilo --check-files "/srv/docs/Spec A.docx" +Path: /srv/docs/Spec A.docx +Verdict: superseded - older revision; latest in group is /srv/docs/Spec B.docx +Revision group: path:/srv/docs/spec.docx + would index /srv/docs/Spec B.docx +``` + +Exit codes make it scriptable: `0` when every enabled directory was scanned +completely (full mode) or the target file would be indexed (path mode); `1` on +a configuration error, an unavailable or incompletely scanned directory, or a +target that would be skipped. `--json` emits the full report as JSON — on +large trees, filter it with `jq` or `grep`. + +## Server Options (`chunksilo-mcp`) + +| Option | Description | +| :--- | :--- | +| `--config` | Path to config.yaml | +| `--transport` | `stdio` or `streamable-http`, overriding `server.transport` (see [configuration.md](configuration.md#server-settings)) | + +The server writes its log to `/mcp.log`, never to the current +directory. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..38352b1 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,267 @@ +# Configuration + +Part of [ChunkSilo](../README.md). + +ChunkSilo uses a single configuration file: `config.yaml` + +## Configuration File + +The file is found in this order: the `--config PATH` argument, the +`CHUNKSILO_CONFIG` environment variable, `./config.yaml`, then +`~/.config/chunksilo/config.yaml`. A path given with `--config` governs the +whole process — every setting is read from that file, not only the ones the +command line happens to mention. + +Edit `config.yaml` to configure your settings: + +```yaml +# Indexing settings - used by chunksilo --build-index +indexing: + directories: + - "./data" + - "/mnt/nfs/shared-docs" + - path: "/mnt/docs/engineering" + include: ["**/*.pdf", "**/*.md"] + exclude: ["**/archive/**"] + chunk_size: 1600 + chunk_overlap: 200 + +# Retrieval settings - used when searching +retrieval: + embed_top_k: 20 + rerank_top_k: 5 + score_threshold: 0.1 + +# Confluence integration (optional) - supports Cloud and Server/Data Center +confluence: + url: "https://confluence.example.com" + username: "your-username" + api_token: "your-api-token" + +# Storage paths (usually don't need to change) +storage: + storage_dir: "./storage" + model_cache_dir: "./models" +``` + +All settings are optional and have sensible defaults. An empty value +(`chunk_size:` with nothing after it) keeps the default rather than setting the +option to nothing. + +## Configuration Reference + +> **Tip:** Run `chunksilo --dump-defaults` to see all available options with their default values. + +### Indexing Settings + +| Setting | Description | +| :--- | :--- | +| `indexing.directories` | List of directories to index (strings or objects) | +| `indexing.chunk_size` | Maximum size of text chunks | +| `indexing.chunk_overlap` | Overlap between adjacent chunks | + +**Per-directory options** (when using object format): + +| Option | Description | +| :--- | :--- | +| `path` | Directory path to index (required) | +| `include` | Glob patterns for files to include | +| `exclude` | Glob patterns for files to exclude | +| `recursive` | Whether to recurse into subdirectories | +| `enabled` | Whether to index this directory | + +**Project-wide directory defaults** — set once instead of repeating per directory: + +| Option | Description | +| :--- | :--- | +| `indexing.defaults.include` | Default include patterns for all directories | +| `indexing.defaults.exclude` | Default exclude patterns for all directories | +| `indexing.defaults.recursive` | Default recursive setting for all directories | +| `indexing.defaults.case_sensitive` | Whether include/exclude globs are case-sensitive (default: false) | + +Use `chunksilo --check-files` to see what these filters do to a tree without +building the index (see [cli.md](cli.md#testing-the-file-filters---check-files)). + +**Advanced indexing options** — performance tuning, timeouts, and diagnostics: + +| Setting | Description | +| :--- | :--- | +| `indexing.parallel_workers` | Threads for parallel file loading (1 = serial) | +| `indexing.batch_size` | Embedding batch size upper bound (adapts down under memory pressure) | +| `indexing.checkpoint_interval_files` | Files processed between index checkpoints | +| `indexing.checkpoint_interval_seconds` | Seconds between index checkpoints | +| `indexing.per_file_seconds` | Give up on a single file after this long (0 disables) | +| `indexing.scan_item_seconds` | Timeout for stat/hash/walk operations while scanning | +| `indexing.slow_file_threshold_seconds` | Warn when one file takes longer than this | + +**Revision handling** — copes with ad-hoc document version control, where +revisions live in revision-named directories (`.../PB3/Spec.docx`, +`.../Rev B/Spec.docx`) or carry a filename token (`Spec PB3.docx`, +`Spec_v2.pdf`): + +| Setting | Description | +| :--- | :--- | +| `indexing.versioning.enabled` | Group revisions of one document and index only the newest (default: true) | +| `indexing.versioning.doc_id_patterns` | Regexes tried against the full path; files sharing an extracted document ID form one group. First capture group (or the whole match) is the ID. Default: empty | +| `indexing.versioning.review_copy_patterns` | Filename markers of review-comment copies, which are never indexed. Setting this replaces the defaults wholesale; `[]` disables detection | +| `indexing.versioning.extra_revision_tokens` | Extra revision-token regex fragments (case-insensitive) added to the built-in grammar | +| `indexing.versioning.index_superseded` | Index older revisions too, stamped and demoted below current documents in results (default: false) | + +How it works: + +- Files are grouped by document ID when a `doc_id_patterns` regex matches + (for example `['\b(DOC-\d{5})\b']` groups every file whose path contains + the same `DOC-12345`); otherwise by their normalized name and location — + revision tokens are stripped from the filename and revision-named directory + components from the path, case-insensitively. The file extension stays part + of the identity (`Spec.pdf` and `Spec.docx` are separate documents), except + that `.doc` and `.docx` count as one. +- The built-in token grammar recognizes preliminary/released letters + (`PA1`, `pb3`, a trailing single uppercase letter like `Spec B.docx`), + `v`-numbers (`v2`, `V1.0`), and `rev`/`revision`/`ver`/`version`/`utg`/`utgåva` + markers. Whole directory components additionally match bare letters and + one-or-two-digit numbers (`B/`, `2/`). Trailing letters after words like + *Appendix*, *Annex*, *Bilaga* are left alone. +- **The newest revision is the one with the newest file modification time.** + Revision labels are never ordered — with no consistent labelling convention, + mtime is the only signal that generalizes. +- Default review-copy markers: `review`, `comment(s)`, `kommentar(er)`, + `granskning`, `copy`, `kopia` (word-bounded, case-insensitive, filename + only). Note these also match documents legitimately named after the + activity ("Literature Review.docx") — `chunksilo --build-index --verbose` + logs every skipped file, and the pattern list is yours to narrow. +- Skipped files are removed from the index on the next `--build-index` and + come back automatically if they become the newest again (for example when + the latest revision is deleted). Set `indexing.versioning.enabled: false` + to restore the old index-everything behavior. + +### Retrieval Settings + +| Setting | Description | +| :--- | :--- | +| `retrieval.embed_model_name` | Embedding model for vector search | +| `retrieval.embed_top_k` | Candidates from vector search before reranking | +| `retrieval.rerank_model_name` | Reranker model | +| `retrieval.rerank_top_k` | Final results after reranking | +| `retrieval.rerank_candidates` | Maximum candidates sent to reranker | +| `retrieval.score_threshold` | Minimum score (0.0-1.0) for results | +| `retrieval.recency_boost` | Recency boost weight (0.0-1.0) | +| `retrieval.recency_half_life_days` | Days until recency boost halves | +| `retrieval.max_chunks_per_doc` | Cap on one document's chunks in the final results, so several documents surface instead of one document's every chunk; freed slots backfill from the next-best chunks (0 disables) | +| `retrieval.offline` | Prevent ML library network requests | + +The score threshold and phrase filters run over the whole reranked candidate +list before the cut to `rerank_top_k`, and raising `retrieval.embed_top_k` +widens the pool that `max_chunks_per_doc` diversifies over, at some reranking +cost. + +### Confluence Settings (optional) + +> **Note:** Confluence support is installed by default; just set the values below to enable it. + +| Setting | Description | +| :--- | :--- | +| `confluence.url` | Confluence base URL (empty = disabled) | +| `confluence.username` | Confluence username | +| `confluence.api_token` | Confluence API token (Cloud) or Personal Access Token (Server/Data Center) | +| `confluence.timeout` | Request timeout in seconds | +| `confluence.max_results` | Maximum results per search | + +**Creating a Confluence API Token:** +1. Log into Confluence +2. Go to Account Settings > Security > API Tokens (for Cloud) or User Profile > Personal Access Tokens (for Server/Data Center) +3. Click "Create API Token" or "Create Token" +4. Copy the token and add it to your config + +### Jira Settings (optional) + +> **Note:** Jira support is installed by default; just set the values below to enable it. + +| Setting | Description | +| :--- | :--- | +| `jira.url` | Jira base URL (empty = disabled) | +| `jira.username` | Jira username/email | +| `jira.api_token` | Jira API token | +| `jira.timeout` | Request timeout in seconds | +| `jira.max_results` | Maximum results per search | +| `jira.projects` | Project keys to search (empty = all) | +| `jira.include_comments` | Include issue comments in search | +| `jira.include_custom_fields` | Include custom fields in search | + +**Creating a Jira API Token:** +1. Log into Jira +2. Go to Account Settings > Security > API Tokens +3. Click "Create API Token" +4. Copy the token and add it to your config + +### SSL Settings (optional) + +| Setting | Description | +| :--- | :--- | +| `ssl.ca_bundle_path` | CA bundle for environments with TLS interception or a private CA: used for model downloads and the Confluence/Jira APIs | + +### Storage Settings + +| Setting | Description | +| :--- | :--- | +| `storage.storage_dir` | Directory for the vector index, the ingestion state, scratch space for `.doc` conversion, and the MCP server log (`mcp.log`) | +| `storage.model_cache_dir` | Directory for model cache | + +Indexing never writes inside an indexed directory: everything it produces lands +under `storage.storage_dir`. + +### Server Settings + +| Setting | Description | +| :--- | :--- | +| `server.transport` | `stdio` (default; the MCP client starts the server itself) or `streamable-http` (listen for remote clients) | +| `server.host` | Bind address for the HTTP transport (default `127.0.0.1`) | +| `server.port` | Port for the HTTP transport (default `8400`) | + +The HTTP transport performs **no authentication of its own**: anyone who can +reach the port can search everything in the index. Keep the loopback bind, or +put a reverse proxy that terminates TLS and authenticates users in front of +the server before exposing it. `chunksilo-mcp --transport streamable-http` +overrides the config for one run, and the HTTP server warms up the index and +models before it accepts its first request. + +### Share Locations (optional) + +| Setting | Description | +| :--- | :--- | +| `shares` | List of `{prefix, unc}` entries. Files under a local mount `prefix` are presented at the share's own location, `unc`, as an `smb://` URI plus the Windows UNC path (see [tools.md](tools.md#share-locations)) | + +```yaml +shares: + - prefix: "/mnt/docs" # local mount point as listed in indexing.directories + unc: "//nas/docs" # the same tree as clients reach it +``` + +Prefixes are matched component-wise on the normalized path (so `/mnt/docs` +does not cover `/mnt/docs-archive`, and `..` cannot escape into another +share), and the longest matching prefix wins. Entries with a relative prefix +or an `unc` that is not `//server/share...` are ignored with a warning. Files +under no prefix keep their `file://` URI. + +## Upgrading from 2.x + +Some indexing options were flattened or removed. A config file that still sets +one of them is loaded (the option is ignored) and a warning names its +replacement: + +| Removed | Use instead | +| :--- | :--- | +| `indexing.timeout.enabled`, `indexing.timeout.per_file_seconds` | `indexing.per_file_seconds` (0 disables) | +| `indexing.timeout.scan_item_seconds` | `indexing.scan_item_seconds` | +| `indexing.timeout.doc_conversion_seconds`, `indexing.timeout.heartbeat_interval_seconds` | internal now | +| `indexing.logging.slow_file_threshold_seconds` | `indexing.slow_file_threshold_seconds` | +| `indexing.logging.log_slow_files` | never read; removed | +| `indexing.enable_parallel_loading` | `indexing.parallel_workers: 1` for serial loading | +| `indexing.enable_adaptive_batching`, `indexing.max_memory_mb` | `indexing.batch_size` is an upper bound; sizing adapts automatically | +| `retrieval.bm25_similarity_top_k` | never read; removed | + +Other behaviour changes: result `file://` URIs are now percent-encoded (a +space is `%20`), each result carries a `unc` field (`null` unless `shares` +covers the file), and by default only the newest revision of a document is +indexed — set `indexing.versioning.enabled: false` for the old +index-everything behaviour. No index rebuild is required. diff --git a/docs/mcp-clients.md b/docs/mcp-clients.md new file mode 100644 index 0000000..434c324 --- /dev/null +++ b/docs/mcp-clients.md @@ -0,0 +1,188 @@ +# MCP Client Configuration + +Part of [ChunkSilo](../README.md). + +Configure your MCP client to run ChunkSilo. Below are examples for common clients. + +> **Note:** Two shapes appear below. **PyPI install** is `pip install chunksilo` +> into a virtualenv you activate, which puts `chunksilo-mcp` on your `PATH` — +> locate it with `which chunksilo-mcp`. **Offline bundle** is the unpacked +> release tarball, which is not on any `PATH`, so give the full +> `/path/to/chunksilo/venv/bin/chunksilo-mcp`. + +## Claude Code + +Add chunksilo as an MCP server using the CLI: + +**PyPI install** (`chunksilo-mcp` on your `PATH`): +```bash +claude mcp add chunksilo --scope user -- chunksilo-mcp --config ~/.config/chunksilo/config.yaml +``` + +**Offline bundle:** +```bash +claude mcp add chunksilo --scope user -- /path/to/chunksilo/venv/bin/chunksilo-mcp --config /path/to/chunksilo/config.yaml +``` + +Verify it's connected: + +```bash +claude mcp list +``` + +## Claude Desktop + +Add to `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows): + +**PyPI install** (`chunksilo-mcp` on your `PATH`): +```json +{ + "mcpServers": { + "chunksilo": { + "command": "chunksilo-mcp", + "args": ["--config", "/path/to/config.yaml"] + } + } +} +``` + +**Offline bundle:** +```json +{ + "mcpServers": { + "chunksilo": { + "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp", + "args": ["--config", "/path/to/chunksilo/config.yaml"] + } + } +} +``` + +## Cline (VS Code Extension) + +Add to `cline_mcp_settings.json` (typically in `~/.config/Code/User/globalStorage/saoudrizwan.claude-dev/settings/`): + +**PyPI install** (`chunksilo-mcp` on your `PATH`): +```json +{ + "mcpServers": { + "chunksilo": { + "command": "chunksilo-mcp", + "args": ["--config", "/path/to/config.yaml"], + "disabled": false, + "autoApprove": [] + } + } +} +``` + +**Offline bundle:** +```json +{ + "mcpServers": { + "chunksilo": { + "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp", + "args": ["--config", "/path/to/chunksilo/config.yaml"], + "disabled": false, + "autoApprove": [] + } + } +} +``` + +## Roo Code (VS Code Extension) + +Add to `mcp_settings.json` (typically in `~/.config/Code/User/globalStorage/rooveterinaryinc.roo-cline/settings/`): + +**PyPI install** (`chunksilo-mcp` on your `PATH`): +```json +{ + "mcpServers": { + "chunksilo": { + "command": "chunksilo-mcp", + "args": ["--config", "/path/to/config.yaml"] + } + } +} +``` + +**Offline bundle:** +```json +{ + "mcpServers": { + "chunksilo": { + "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp", + "args": ["--config", "/path/to/chunksilo/config.yaml"] + } + } +} +``` + +## opencode + +Add an `mcp` entry to your opencode config — `~/.config/opencode/opencode.json` for +all projects, or `opencode.json` in a project root for just that one. Config files +are merged, with the project file taking precedence. + +**PyPI install** (`chunksilo-mcp` on your `PATH`): +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "chunksilo": { + "type": "local", + "command": ["chunksilo-mcp", "--config", "/home/you/.config/chunksilo/config.yaml"], + "enabled": true + } + } +} +``` + +**Offline bundle** — use the absolute path to the bundle's binary: +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "chunksilo": { + "type": "local", + "command": [ + "/path/to/chunksilo/venv/bin/chunksilo-mcp", + "--config", "/path/to/chunksilo/config.yaml" + ], + "enabled": true + } + } +} +``` + +Note that `command` is an array — the executable and each argument are separate +elements, not one string. Start opencode and ask it something answerable from your +documents to confirm the `search_docs` tool is being called. + +## Remote server over HTTP + +With `server.transport: streamable-http` (or `chunksilo-mcp --transport +streamable-http`) the server listens on `server.host:server.port` — by default +`http://127.0.0.1:8400/mcp` — and any MCP client that speaks streamable HTTP +can connect to it. ChunkSilo authenticates nobody on this transport, so expose +it only behind a reverse proxy that does; the examples below use the loopback +address. + +**Claude Code:** +```bash +claude mcp add --transport http chunksilo http://127.0.0.1:8400/mcp +``` + +**opencode:** +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "chunksilo": { + "type": "remote", + "url": "http://127.0.0.1:8400/mcp", + "enabled": true + } + } +} +``` diff --git a/docs/tools.md b/docs/tools.md new file mode 100644 index 0000000..5b2ed49 --- /dev/null +++ b/docs/tools.md @@ -0,0 +1,77 @@ +# MCP Tools + +Part of [ChunkSilo](../README.md). + +ChunkSilo exposes one MCP tool, `search_docs`, which finds relevant passages +across everything that was indexed (plus live Confluence and Jira results when +those integrations are configured). + +## Example `search_docs` output + +```json +{ + "matched_files": [ + { + "uri": "file:///docs/database-configuration.docx", + "unc": null, + "score": 0.8432 + } + ], + "num_matched_files": 1, + "chunks": [ + { + "text": "To configure the database connection, set the DATABASE_URL environment variable...", + "score": 0.912, + "location": { + "uri": "file:///docs/setup-guide.pdf", + "unc": null, + "page": 12, + "line": null, + "heading_path": ["Getting Started", "Configuration", "Database"] + } + } + ], + "num_chunks": 1, + "query": "how to configure the database", + "retrieval_time": "0.42s" +} +``` + +- `matched_files` — files whose *names* matched the query, one entry per + document (the newest revision of a document group), at most five. +- `chunks` — the passages that carry the answer, in rank order. At most + `retrieval.max_chunks_per_doc` chunks come from any one document, so the + list spans documents instead of one document's every chunk. +- `location.uri` — a percent-encoded `file://` URI for a local file, the page + URL for a Confluence result, or the issue URL for a Jira result. +- `location.unc` — the Windows UNC path of a file on a mapped network share + (see below); `null` otherwise. +- `location.page` / `location.line` — the page number (PDF, DOCX) or line + number (Markdown, text) the chunk starts on. +- `location.heading_path` — the section hierarchy the chunk sits under. + +Errors are reported as an `error` field with a generic message; the detail is +in the server log. + +## Share locations + +A file that lives under one of the `shares` prefixes in the configuration is +presented at its real network location — an `smb://` URI plus the Windows UNC +path — never at the local mount path, which means nothing to a client on +another machine: + +```json +{ + "uri": "smb://nas/docs/setup-guide.pdf", + "unc": "\\\\nas\\docs\\setup-guide.pdf", + "page": 12, + "line": null, + "heading_path": ["Getting Started", "Configuration", "Database"] +} +``` + +The `smb://` form is percent-encoded like a URL; the UNC form is raw, because +Windows opens `\\server\share\...` verbatim. Anything not covered by a share +mapping (including everything in a single-machine setup, which typically +configures none) keeps a `file:///...` URI, which the same-machine client can +open directly. See [configuration.md](configuration.md#share-locations-optional). diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..2e98f06 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,19 @@ +# Troubleshooting + +Part of [ChunkSilo](../README.md). + +## Common issues + +- **Index missing**: Run `chunksilo --build-index` (PyPI install) or `./venv/bin/chunksilo --build-index` (offline bundle). +- **Retrieval errors**: Check paths in your MCP client configuration. The MCP server's own log is `/mcp.log`. +- **Offline mode**: PyPI installs default to `offline: false` (models auto-download). The offline bundle includes pre-downloaded models and sets `offline: true`. Set `retrieval.offline: true` in `config.yaml` to prevent network calls after initial model download. +- **Confluence Integration**: Included by default — set `confluence.url`, `confluence.username`, and `confluence.api_token` in `config.yaml`. +- **Jira Integration**: Included by default — set `jira.url`, `jira.username`, and `jira.api_token` in `config.yaml`. Optionally configure `jira.projects` to restrict search to specific project keys. +- **Custom CA Bundle**: Set `ssl.ca_bundle_path` in `config.yaml` for custom certificates. +- **Network mounts**: Unavailable directories are skipped with a warning; indexing continues with available directories. Files already indexed from a directory that could not be scanned — or individual files that could not be read — are **kept**, not treated as deleted, and the run reports how many (`N unseen (kept)`). Otherwise a mount that is briefly slow drops its whole tree from the index and the next run re-indexes it from scratch. The consequence is that content genuinely removed from a still-configured directory that has become unreachable stays in the index; to drop it, remove the directory from `indexing.directories` and rebuild. +- **Files that keep being re-indexed**: run `chunksilo --build-index --verbose` and look for `Reprocessing :` — it names the reason (`content changed`, `revision status changed`). Files with no extractable text (an image-only `.docx`, for instance) are recorded as empty and are not retried until they change; files that failed to load are reported separately and *are* retried. +- **A file is not being indexed**: run `chunksilo --check-files ` — it names the exact reason (excluded by a glob pattern, no include pattern matched, pruned directory, superseded revision, review copy, disabled or unavailable directory) without re-indexing, and `chunksilo --check-files` alone reports a verdict for every file the scan sees (see [cli.md](cli.md#testing-the-file-filters---check-files)). By default only the newest revision of a document group is indexed, and review-comment copies never are; during a real build the scan summary counts them (`N superseded (skipped)`, `N review copies (skipped)`) and `--verbose` names each file. If the classification is wrong, adjust `indexing.versioning.review_copy_patterns` / `doc_id_patterns`, set `indexing.versioning.index_superseded: true` to keep old revisions searchable, or set `indexing.versioning.enabled: false` to turn revision handling off entirely (see [configuration.md](configuration.md)). +- **Results all come from one document**: `retrieval.max_chunks_per_doc` (default 2) caps how many chunks of one document appear; raise `retrieval.embed_top_k` to widen the pool it chooses from. +- **Legacy .doc files**: Requires LibreOffice to be installed for automatic conversion to .docx. If LibreOffice is not found, .doc files are skipped with a warning. Full heading extraction is supported. Conversion runs on a copy of the file in a private directory under `storage.storage_dir`, so nothing is ever written next to the original. +- **Old config options**: after upgrading from 2.x, a warning such as `'indexing.timeout' is no longer used and is ignored; use ...` means the file still sets a removed option; see the [upgrade notes](configuration.md#upgrading-from-2x). +- **HTTP transport**: `chunksilo-mcp --transport streamable-http` binds `127.0.0.1:8400` by default and authenticates nobody. If a remote client cannot connect, that is the loopback bind doing its job — put an authenticating reverse proxy in front and point `server.host` at the address the proxy uses, rather than exposing the port directly. diff --git a/scripts/prepare-common.sh b/scripts/prepare-common.sh index 70cef0a..5bcd98d 100755 --- a/scripts/prepare-common.sh +++ b/scripts/prepare-common.sh @@ -27,6 +27,7 @@ cp -r src "$COMMON_ROOT/" cp pyproject.toml "$COMMON_ROOT/" cp requirements.txt "$COMMON_ROOT/" cp README.md "$COMMON_ROOT/" +cp -r docs "$COMMON_ROOT/" cp LICENSE "$COMMON_ROOT/" cp NOTICE "$COMMON_ROOT/" cp config.yaml "$COMMON_ROOT/" diff --git a/test/README.md b/test/README.md index 114a2a8..415496c 100644 --- a/test/README.md +++ b/test/README.md @@ -9,14 +9,14 @@ This directory contains all test-related files for ChunkSilo. - `test_rag_metrics.py` - RAG metrics test suite with evaluation metrics - `create_test_doc.py` - Utility script to create test documents - `TESTING.md` - Comprehensive testing documentation -- `requirements.txt` - Test-only dependencies (not included in release package) +- Test-only dependencies come from the `test` extra in `pyproject.toml` (`pip install -e '.[test]'`) ## Installation Install test dependencies: ```bash -pip install -r test/requirements.txt +pip install -e '.[test]' ``` ## Running Tests @@ -30,7 +30,7 @@ OFFLINE=0 python test_rag_metrics.py ## Important Notes -- **Test files are excluded from the release package**: The release workflow only includes production files (`chunksilo.py`, `index.py`, `requirements.txt`, etc.). Test files remain in the repository for development but are not packaged for end users. +- **Test files are excluded from the release package**: The release workflow only includes production files (`src/chunksilo/`, `requirements.txt`, `config.yaml`, `docs/`, etc.). Test files remain in the repository for development but are not packaged for end users. - **Tests require online access**: The RAG metrics test suite downloads documents from the web, so run with `OFFLINE=0`. - **Test artifacts are gitignored**: Generated test data, storage, and results are in `.gitignore` to keep the repository clean. diff --git a/test/TESTING.md b/test/TESTING.md index b6cd10c..455fbef 100644 --- a/test/TESTING.md +++ b/test/TESTING.md @@ -49,7 +49,7 @@ First, install test dependencies: ```bash source .venv/bin/activate -pip install -r test/requirements.txt +pip install -e '.[test]' ``` ### 1. Basic functionality (ingestion + index loading) @@ -194,7 +194,7 @@ You can also run the MCP server directly: ```bash source .venv/bin/activate -python chunksilo.py +chunksilo-mcp ``` The server speaks MCP over stdio and is normally launched by an MCP client (such as Continue). Running it manually is useful for debugging, but you'll need an MCP-aware client to actually send tool calls. diff --git a/test/run_tests.sh b/test/run_tests.sh index 488f5cd..2c478c0 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -10,7 +10,7 @@ # # Requirements: # - Python 3.11 -# - Dependencies from requirements.txt and test/requirements.txt +# - The package installed with its test extra: pip install -e '.[test]' set -e @@ -29,7 +29,7 @@ else echo "Error: Python 3.11 is required but not found." echo "Please create a virtual environment:" echo " python3.11 -m venv .venv311" - echo " .venv311/bin/pip install -r requirements.txt -r test/requirements.txt" + echo " .venv311/bin/pip install -e '.[test]'" exit 1 fi From cc56e803fc7b80ba14cdfc94aefa68ae4bf00927 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Sun, 13 Sep 2026 22:01:17 +0200 Subject: [PATCH 15/18] feat: make the HTTP transport usable behind an authenticating reverse proxy Behind a reverse proxy the MCP transport rejected every request: the SDK enables DNS-rebinding protection for a loopback bind and then accepts only loopback Host headers (421 otherwise, 403 for a foreign Origin), while a proxy forwards the public hostname. - server.public_url names the URL the proxy serves the instance on; its host and origin are added to the transport's allow-list, and server.allowed_hosts / allowed_origins take any further names. Without them the SDK's loopback-only default is kept. - server.authorization_servers publishes RFC 9728 protected-resource metadata at /.well-known/oauth-protected-resource (and the path-suffixed form) naming the authorization server MCP clients should obtain a token from. ChunkSilo only publishes the document; verifying tokens remains the proxy's job. - GET /health answers once the server is warmed up and listening. - A forwarded X-Forwarded-User / X-Forwarded-Email is logged with each search, for an audit trail per instance; it is never used for a decision. docs/reverse-proxy.md walks through one instance per body of documents behind oauth2-proxy, with membership of a KeyCloak group deciding who may search it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- AGENTS.md | 2 +- README.md | 3 +- config.yaml | 4 ++ docs/configuration.md | 12 +++- docs/mcp-clients.md | 5 +- docs/reverse-proxy.md | 138 ++++++++++++++++++++++++++++++++++++++ docs/troubleshooting.md | 3 +- src/chunksilo/cfgload.py | 15 +++++ src/chunksilo/server.py | 141 +++++++++++++++++++++++++++++++++++++-- test/test_server.py | 135 +++++++++++++++++++++++++++++++++++++ 10 files changed, 444 insertions(+), 14 deletions(-) create mode 100644 docs/reverse-proxy.md diff --git a/AGENTS.md b/AGENTS.md index 19d960f..db56ca9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,4 +43,4 @@ test/run_tests.sh ## Search Invariants - **Result formatting must not touch the filesystem.** Result paths often live on a network mount where a stat plus a readlink per component dominates the whole query; `_normalize_result_path` and the share-location mapping (`shareuri.py`) work from the stored string and the configuration alone. Resolving would also rewrite a path into the mount's internal form, which no application can open. `test/test_chunk_location.py` pins this. - **The lazy loaders are double-checked-locked.** `_ensure_embed_model`, `_ensure_reranker`, `_ensure_bm25_retriever` and `load_llamaindex_index` are process-wide; over the HTTP transport tool calls run concurrently in a thread pool with no session affinity, so without the locks each first caller cold-starts its own copy. `run_server` calls `search.warm_up` before serving HTTP. -- **The HTTP transport authenticates nobody.** Access control belongs to a reverse proxy in front of the server; do not add credentials, tokens or per-user filtering to the transport itself. +- **The HTTP transport authenticates nobody.** Access control belongs to a reverse proxy in front of the server; do not add credentials, tokens or per-user filtering to the transport itself. `server.public_url` / `allowed_hosts` only widen the SDK's Host/Origin allow-list, `server.authorization_servers` only publishes discovery metadata, and a forwarded `X-Forwarded-User` is logged, never acted on. diff --git a/README.md b/README.md index e48d444..8b160b6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ChunkSilo is like a local Google for your documents. It uses semantic search — - Heading-aware results with source links back to the original file, spread across documents instead of one document's every chunk - Date filtering and recency boosting - Files indexed from a mounted network share can be presented at the share's own location (an `smb://` URI plus the Windows UNC path) -- Serves MCP over stdio to a local client, or over streamable-http behind a reverse proxy of your own +- Serves MCP over stdio to a local client, or over streamable-http behind a reverse proxy of your own (oauth2-proxy + KeyCloak walkthrough included) - Optional Confluence and Jira integrations (supports Cloud and Server/Data Center) ## Installation @@ -124,6 +124,7 @@ mounted network share are presented. - [docs/cli.md](docs/cli.md) — `chunksilo` and `chunksilo-mcp` options, including `--check-files` for explaining why a file is or is not indexed. - [docs/tools.md](docs/tools.md) — the `search_docs` result shape. - [docs/mcp-clients.md](docs/mcp-clients.md) — client setup for Claude Code, Claude Desktop, Cline, Roo Code and opencode, over stdio or HTTP. +- [docs/reverse-proxy.md](docs/reverse-proxy.md) — sharing an instance behind an authenticating reverse proxy, with oauth2-proxy and KeyCloak group-based access. - [docs/troubleshooting.md](docs/troubleshooting.md) — indexing surprises, network mounts, offline mode and more. ## License diff --git a/config.yaml b/config.yaml index c3f0a60..5707f77 100644 --- a/config.yaml +++ b/config.yaml @@ -114,6 +114,10 @@ storage: # transport: "stdio" # stdio | streamable-http # host: "127.0.0.1" # port: 8400 +# public_url: "https://search.example.com" # the URL the proxy serves this instance on +# allowed_hosts: [] # further Host values to accept ("host:*" = any port) +# allowed_origins: [] # further Origin values to accept +# authorization_servers: [] # e.g. ["https://keycloak.example.com/realms/docs"] # Network shares (optional): present files indexed from a mounted share at the # share's own location - an smb:// URI plus the Windows UNC path - instead of diff --git a/docs/configuration.md b/docs/configuration.md index 38352b1..8980b00 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -217,13 +217,19 @@ under `storage.storage_dir`. | `server.transport` | `stdio` (default; the MCP client starts the server itself) or `streamable-http` (listen for remote clients) | | `server.host` | Bind address for the HTTP transport (default `127.0.0.1`) | | `server.port` | Port for the HTTP transport (default `8400`) | +| `server.public_url` | The URL a reverse proxy serves this instance on, e.g. `https://search.example.com`. Required behind a proxy: the transport rejects any `Host` header it does not expect with `421`, and this adds the public hostname (and origin) to what it accepts | +| `server.allowed_hosts` | Further `Host` header values to accept (`host` or `host:port`; `host:*` for any port) | +| `server.allowed_origins` | Further `Origin` values to accept (`scheme://host[:port]`; `scheme://host:*` for any port) | +| `server.authorization_servers` | Issuer URLs (e.g. a KeyCloak realm) published as RFC 9728 protected-resource metadata at `/.well-known/oauth-protected-resource`, so MCP clients can discover where to obtain a token. Requires `public_url`. ChunkSilo never verifies tokens itself | The HTTP transport performs **no authentication of its own**: anyone who can reach the port can search everything in the index. Keep the loopback bind, or put a reverse proxy that terminates TLS and authenticates users in front of -the server before exposing it. `chunksilo-mcp --transport streamable-http` -overrides the config for one run, and the HTTP server warms up the index and -models before it accepts its first request. +the server before exposing it — see [reverse-proxy.md](reverse-proxy.md) for +an oauth2-proxy and KeyCloak setup. `chunksilo-mcp --transport streamable-http` +overrides the config for one run, the HTTP server warms up the index and +models before it accepts its first request, and `GET /health` answers once it +is listening. ### Share Locations (optional) diff --git a/docs/mcp-clients.md b/docs/mcp-clients.md index 434c324..013edd8 100644 --- a/docs/mcp-clients.md +++ b/docs/mcp-clients.md @@ -165,8 +165,9 @@ With `server.transport: streamable-http` (or `chunksilo-mcp --transport streamable-http`) the server listens on `server.host:server.port` — by default `http://127.0.0.1:8400/mcp` — and any MCP client that speaks streamable HTTP can connect to it. ChunkSilo authenticates nobody on this transport, so expose -it only behind a reverse proxy that does; the examples below use the loopback -address. +it only behind a reverse proxy that does — [reverse-proxy.md](reverse-proxy.md) +walks through an oauth2-proxy and KeyCloak setup. The examples below use the +loopback address. **Claude Code:** ```bash diff --git a/docs/reverse-proxy.md b/docs/reverse-proxy.md new file mode 100644 index 0000000..34e5f42 --- /dev/null +++ b/docs/reverse-proxy.md @@ -0,0 +1,138 @@ +# Running behind a reverse proxy + +Part of [ChunkSilo](../README.md). + +ChunkSilo's HTTP transport authenticates nobody. The intended way to share an +instance is to put an authenticating reverse proxy in front of it and let the +proxy decide who gets through. This page shows that setup with +[oauth2-proxy](https://oauth2-proxy.github.io/oauth2-proxy/) checking users +against a KeyCloak realm, one ChunkSilo instance per body of documents, and +membership of a KeyCloak group deciding who may search that instance. + +```text +MCP client ──HTTPS──▶ oauth2-proxy (KeyCloak login, group check) ──▶ chunksilo-mcp + https://search.example.com 127.0.0.1:8400 +``` + +Run one `chunksilo-mcp` and one oauth2-proxy per information domain: each +oauth2-proxy has a single allowed-group setting, so the pair is the unit of +access control. Give each instance its own `config.yaml` (its own +`indexing.directories`, `storage.storage_dir` and `server.port`). + +## ChunkSilo side + +```yaml +server: + transport: "streamable-http" + host: "127.0.0.1" # only the proxy can reach it + port: 8400 + public_url: "https://search.example.com" # what the proxy serves this instance on + authorization_servers: # where MCP clients get a token + - "https://keycloak.example.com/realms/docs" +``` + +- `public_url` is required behind a proxy. The MCP transport rejects requests + whose `Host` header is not the bind address (`421 Invalid Host header`), and + a proxy forwards the public hostname. `allowed_hosts` / `allowed_origins` + add further values if the proxy is reached under more than one name. +- `authorization_servers` makes ChunkSilo serve the standard + protected-resource metadata at `/.well-known/oauth-protected-resource` + (and `/.well-known/oauth-protected-resource/mcp`), naming the KeyCloak realm. + MCP clients that obtain their own token read it to find where to log in. + ChunkSilo only publishes the document; it never checks a token. +- `GET /health` answers `{"status": "ok"}` once the server is warmed up and + listening; use it for readiness checks. +- If the proxy forwards `X-Forwarded-User` or `X-Forwarded-Email`, each search + is logged with that identity in `/mcp.log`. The headers are + trusted only because the loopback bind means only the proxy can set them; + they never affect what a search returns. + +Start it with `chunksilo-mcp --config /etc/chunksilo/docs.yaml`, or pass +`--transport streamable-http` to override a config that says `stdio`. + +## oauth2-proxy side + +The essential settings, as flags (every one has a config-file and +environment-variable form): + +```bash +oauth2-proxy \ + --http-address=127.0.0.1:4180 \ # put TLS termination in front, or use --tls-cert-file + --upstream=http://127.0.0.1:8400/ \ + --provider=keycloak-oidc \ + --oidc-issuer-url=https://keycloak.example.com/realms/docs \ + --client-id=chunksilo-docs --client-secret=... \ + --redirect-url=https://search.example.com/oauth2/callback \ + --email-domain='*' \ + --allowed-group=/docs-readers \ # the KeyCloak group that may search this instance + --api-routes='^/mcp' \ # JSON-RPC callers get 401, not a login redirect + --skip-jwt-bearer-tokens=true \ # accept KeyCloak-issued bearer tokens + --skip-auth-route='GET=^/health' \ + --skip-auth-route='GET=^/\.well-known/oauth-protected-resource' \ + --pass-user-headers=true \ # X-Forwarded-User / -Email for the search log + --cookie-secret=... +``` + +Two kinds of client then work: + +- **Browser-driven sessions** log in through KeyCloak and carry the + oauth2-proxy cookie. +- **MCP clients that do OAuth themselves** (Claude Code, opencode and other + clients that support remote servers with authorization) get a token from + KeyCloak and send it as a bearer token; `--skip-jwt-bearer-tokens` + validates it against the realm and `--allowed-group` applies to it as well. + The token must carry the `groups` claim and an audience oauth2-proxy accepts + (`--oidc-extra-audience` if it is not the client id). + +Unauthenticated `/mcp` calls receive a plain 401 from oauth2-proxy. Clients +that follow the MCP authorization flow then read the protected-resource +metadata to find the realm; if a client insists on the `WWW-Authenticate` +header naming that metadata, add it on the TLS terminator (nginx: +`add_header WWW-Authenticate 'Bearer resource_metadata="https://search.example.com/.well-known/oauth-protected-resource"' always;` +on the 401 response). + +## KeyCloak side + +- A confidential client for oauth2-proxy (`chunksilo-docs` above) with the + callback URL as a valid redirect URI. +- A **Group Membership** mapper that puts the user's groups into the tokens as + a `groups` claim; oauth2-proxy compares `--allowed-group` against it. Group + names in KeyCloak are paths (`/docs-readers`). +- For MCP clients that log in by themselves: either allow dynamic client + registration in the realm (client registration policies) or create a public + client with PKCE and the redirect URIs those clients use, and map the + audience so the token is accepted by oauth2-proxy. + +## Client side + +Point the client at the proxy's URL, not at ChunkSilo directly: + +```bash +claude mcp add --transport http chunksilo https://search.example.com/mcp +``` + +```json +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "chunksilo": { + "type": "remote", + "url": "https://search.example.com/mcp", + "enabled": true + } + } +} +``` + +## Checking the deployment + +1. `curl https://search.example.com/health` returns `{"status":"ok"}`. +2. `curl https://search.example.com/.well-known/oauth-protected-resource` + returns the realm URL. +3. An unauthenticated `POST https://search.example.com/mcp` returns 401. +4. `/mcp.log` on the ChunkSilo host shows `Serving MCP over + streamable-http ... public URL https://search.example.com` at startup and + `Search by ` lines afterwards. +5. A `421 Invalid Host header` in the client, or `Invalid Host header:` in + `mcp.log`, means `server.public_url` does not match the hostname the proxy + forwards. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 2e98f06..60bec4d 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -16,4 +16,5 @@ Part of [ChunkSilo](../README.md). - **Results all come from one document**: `retrieval.max_chunks_per_doc` (default 2) caps how many chunks of one document appear; raise `retrieval.embed_top_k` to widen the pool it chooses from. - **Legacy .doc files**: Requires LibreOffice to be installed for automatic conversion to .docx. If LibreOffice is not found, .doc files are skipped with a warning. Full heading extraction is supported. Conversion runs on a copy of the file in a private directory under `storage.storage_dir`, so nothing is ever written next to the original. - **Old config options**: after upgrading from 2.x, a warning such as `'indexing.timeout' is no longer used and is ignored; use ...` means the file still sets a removed option; see the [upgrade notes](configuration.md#upgrading-from-2x). -- **HTTP transport**: `chunksilo-mcp --transport streamable-http` binds `127.0.0.1:8400` by default and authenticates nobody. If a remote client cannot connect, that is the loopback bind doing its job — put an authenticating reverse proxy in front and point `server.host` at the address the proxy uses, rather than exposing the port directly. +- **HTTP transport**: `chunksilo-mcp --transport streamable-http` binds `127.0.0.1:8400` by default and authenticates nobody. If a remote client cannot connect, that is the loopback bind doing its job — put an authenticating reverse proxy in front (see [reverse-proxy.md](reverse-proxy.md)) rather than exposing the port directly. +- **`421 Invalid Host header` behind a proxy**: the MCP transport only accepts the `Host` values it expects, and a proxy forwards the public hostname. Set `server.public_url` to the URL the proxy serves (add `server.allowed_hosts` for any further names). A `403 Invalid Origin header` is the same check for browser-based clients: `server.allowed_origins`. diff --git a/src/chunksilo/cfgload.py b/src/chunksilo/cfgload.py index ec2c465..92d6340 100644 --- a/src/chunksilo/cfgload.py +++ b/src/chunksilo/cfgload.py @@ -165,6 +165,21 @@ def _find_config() -> Path: # with TLS termination and a login layer in front of this server. "host": "127.0.0.1", "port": 8400, + # The URL clients reach this server on when a reverse proxy sits in + # front, e.g. "https://search.example.com". The MCP transport rejects + # any Host header other than the bind address unless it is allowed + # here, and a proxy forwards the public hostname. Empty = only the + # bind address is accepted. + "public_url": "", + # Further Host header values and Origins to accept ("host:*" = any port). + "allowed_hosts": [], + "allowed_origins": [], + # Issuer URLs of the authorization servers (e.g. a KeyCloak realm) MCP + # clients should obtain a token from. Published as RFC 9728 + # protected-resource metadata at /.well-known/oauth-protected-resource + # so clients can discover them; this server never verifies tokens - + # that is the reverse proxy's job. Empty = the metadata is not served. + "authorization_servers": [], }, # Present files indexed from a mounted network share at the share's own # location (an smb:// URI plus the Windows UNC path) instead of the diff --git a/src/chunksilo/server.py b/src/chunksilo/server.py index adb47fb..4b228fc 100644 --- a/src/chunksilo/server.py +++ b/src/chunksilo/server.py @@ -10,7 +10,9 @@ - streamable-http: listens on server.host:server.port for any MCP client that can reach it. This server performs NO authentication of its own - keep the loopback bind (the default) or put an authenticating reverse proxy in front - of it before exposing the port. + of it before exposing the port. server.public_url tells the transport which + Host header that proxy forwards; server.authorization_servers lets clients + discover where to obtain a token. """ import argparse import asyncio @@ -19,9 +21,17 @@ import os from pathlib import Path from typing import Annotated, Any +from urllib.parse import urlparse -from mcp.server.fastmcp import FastMCP +from mcp.server.fastmcp import Context, FastMCP +from mcp.server.transport_security import TransportSecuritySettings from pydantic import Field +from starlette.requests import Request +from starlette.responses import JSONResponse + +from . import __version__ + +logger = logging.getLogger(__name__) # Log file configuration. The log lives in the storage directory - never the # CWD, which on a mis-started service could be inside an indexed tree - and @@ -32,6 +42,10 @@ TRANSPORTS = ("stdio", "streamable-http") +# What the MCP SDK accepts for a loopback bind; kept when a public URL is added. +_LOOPBACK_HOSTS = ["127.0.0.1:*", "localhost:*", "[::1]:*"] +_LOOPBACK_ORIGINS = ["http://127.0.0.1:*", "http://localhost:*", "http://[::1]:*"] + # Module-level state (set during initialization) _server_config_path: Path | None = None _mcp: FastMCP | None = None @@ -69,12 +83,84 @@ def _setup_logging(storage_dir: str | Path): logging.getLogger("llama_index.readers.confluence").setLevel(logging.WARNING) +def _public_url(server_cfg: dict[str, Any]) -> tuple[str, str, str] | None: + """``(url, scheme, netloc)`` from server.public_url, or None when unset.""" + raw = str(server_cfg.get("public_url") or "").strip().rstrip("/") + if not raw: + return None + parsed = urlparse(raw) + if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.path: + raise SystemExit( + "server.public_url must be an absolute URL with no path, such as " + f"https://search.example.com - got {raw!r}" + ) + return raw, parsed.scheme, parsed.netloc + + +def _dedupe(values: list[str]) -> list[str]: + return list(dict.fromkeys(v for v in values if v)) + + +def _transport_security(server_cfg: dict[str, Any]) -> TransportSecuritySettings | None: + """Host/Origin allow-list for the HTTP transport, or None for the SDK default. + + The SDK enables DNS-rebinding protection for a loopback bind and then + accepts only loopback Host headers (421 otherwise). A reverse proxy forwards + the public hostname, so the public URL - and anything in allowed_hosts / + allowed_origins - has to be allowed explicitly; the loopback entries stay so + the proxy can also address the upstream directly. + """ + public = _public_url(server_cfg) + extra_hosts = [str(h) for h in server_cfg.get("allowed_hosts") or []] + extra_origins = [str(o) for o in server_cfg.get("allowed_origins") or []] + if public is None and not extra_hosts and not extra_origins: + return None + + hosts = list(_LOOPBACK_HOSTS) + origins = list(_LOOPBACK_ORIGINS) + if public is not None: + _url, scheme, netloc = public + parsed = urlparse(f"{scheme}://{netloc}") + host_only = netloc[: -(len(str(parsed.port)) + 1)] if parsed.port else netloc + hosts += [netloc, host_only, f"{host_only}:*"] + origins += [f"{scheme}://{netloc}", f"{scheme}://{host_only}", f"{scheme}://{host_only}:*"] + bind_host = str(server_cfg.get("host") or "") + if bind_host and bind_host not in ("127.0.0.1", "localhost", "::1", "0.0.0.0", "::"): + hosts.append(f"{bind_host}:*") + hosts += extra_hosts + origins += extra_origins + return TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=_dedupe(hosts), + allowed_origins=_dedupe(origins), + ) + + +def _forwarded_user(ctx: Context | None) -> str | None: + """Identity a reverse proxy forwarded (X-Forwarded-User / -Email), if any. + + Trusted only because the HTTP bind is the proxy's private upstream, and used + for the log line alone - never for an authorization decision. + """ + if ctx is None: + return None + try: + request = ctx.request_context.request + except (ValueError, AttributeError): + return None + if request is None: + return None + headers = request.headers + return headers.get("x-forwarded-user") or headers.get("x-forwarded-email") or None + + def _create_server( config: dict[str, Any] | None = None, transport: str = "stdio" ) -> FastMCP: """Create and configure the MCP server with tools. - For ``streamable-http`` the bind address comes from ``config["server"]``; + For ``streamable-http`` the bind address, the accepted Host/Origin values + and the optional protected-resource metadata come from ``config["server"]``; the stdio server needs no configuration at all. """ from .search import run_search @@ -89,7 +175,9 @@ def _create_server( # never depends on which worker served the previous one. stateless_http=True, json_response=True, + transport_security=_transport_security(server_cfg), ) + _add_http_routes(mcp, server_cfg) else: mcp = FastMCP("chunksilo") @@ -98,8 +186,12 @@ async def search_docs( query: Annotated[str, Field(description="Search query text")], date_from: Annotated[str | None, Field(description="Optional start date filter (YYYY-MM-DD format, inclusive)")] = None, date_to: Annotated[str | None, Field(description="Optional end date filter (YYYY-MM-DD format, inclusive)")] = None, + ctx: Context | None = None, ) -> dict[str, Any]: """Search across all your indexed documentation using a natural language query.""" + user = _forwarded_user(ctx) + if user: + logger.info("Search by %s", user) loop = asyncio.get_running_loop() return await loop.run_in_executor( None, lambda: run_search(query, date_from, date_to, config_path=_server_config_path) @@ -108,12 +200,47 @@ async def search_docs( return mcp +def _add_http_routes(mcp: FastMCP, server_cfg: dict[str, Any]) -> None: + """Plain HTTP routes next to /mcp: a health check and, when configured, + the RFC 9728 protected-resource metadata MCP clients use to find the + authorization server. Neither is authenticated - exempt them at the proxy.""" + + @mcp.custom_route("/health", methods=["GET"]) + async def health(_request: Request) -> JSONResponse: + # The server only starts listening after warm-up, so reachable = ready. + return JSONResponse({"status": "ok", "version": __version__}) + + authorization_servers = [str(u) for u in server_cfg.get("authorization_servers") or []] + if not authorization_servers: + return + public = _public_url(server_cfg) + if public is None: + raise SystemExit( + "server.authorization_servers requires server.public_url: the metadata " + "must name the URL clients reach this server on" + ) + mcp_path = mcp.settings.streamable_http_path + metadata = { + "resource": public[0] + mcp_path, + "authorization_servers": authorization_servers, + "bearer_methods_supported": ["header"], + "resource_name": "ChunkSilo", + } + + async def protected_resource(_request: Request) -> JSONResponse: + return JSONResponse(metadata) + + # Both the origin-wide form and the path-suffixed form of the well-known + # URL, so a client can look it up either way. + for path in ("/.well-known/oauth-protected-resource", + "/.well-known/oauth-protected-resource" + mcp_path): + mcp.custom_route(path, methods=["GET"])(protected_resource) + + def run_server(config_path: Path | None = None, transport_override: str | None = None): """Start the MCP server.""" global _server_config_path, _mcp - logger = logging.getLogger(__name__) - from .cfgload import load_config # Load - and thereby activate - the configuration before importing @@ -147,10 +274,12 @@ def run_server(config_path: Path | None = None, transport_override: str | None = search_module.warm_up(config) _mcp = _create_server(config, transport) + public = _public_url(config["server"]) logger.info( - "Serving MCP over streamable-http on %s:%s (no authentication - front it with a proxy before exposing it)", + "Serving MCP over streamable-http on %s:%s%s (no authentication - front it with a proxy before exposing it)", config["server"]["host"], config["server"]["port"], + f", public URL {public[0]}" if public else "", ) _mcp.run(transport="streamable-http") diff --git a/test/test_server.py b/test/test_server.py index 6088e08..13298cb 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -211,3 +211,138 @@ def test_unknown_transport_exits(self, monkeypatch, tmp_path): with pytest.raises(SystemExit): self._run(config, monkeypatch, tmp_path) + + +# ============================================================================= +# Tests for running behind a reverse proxy +# ============================================================================= + + +_INIT = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "test", "version": "0"}, + }, +} +_SEARCH = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "search_docs", "arguments": {"query": "widget"}}, +} + + +def _http_config(**server): + base = {"transport": "streamable-http", "host": "127.0.0.1", "port": 8765} + return {"server": {**base, **server}} + + +class TestReverseProxySupport: + def test_without_public_url_the_sdk_loopback_protection_stays(self): + security = _create_server(_http_config(), "streamable-http").settings.transport_security + + assert security.enable_dns_rebinding_protection is True + assert "127.0.0.1:*" in security.allowed_hosts + assert "search.example.com" not in security.allowed_hosts + + def test_public_url_host_and_origin_are_accepted(self): + config = _http_config(public_url="https://search.example.com") + + security = _create_server(config, "streamable-http").settings.transport_security + + assert {"search.example.com", "search.example.com:*", "127.0.0.1:*"} <= set(security.allowed_hosts) + assert {"https://search.example.com", "https://search.example.com:*"} <= set(security.allowed_origins) + + def test_public_url_with_a_port(self): + config = _http_config(public_url="https://search.example.com:8443/") + + security = _create_server(config, "streamable-http").settings.transport_security + + assert {"search.example.com:8443", "search.example.com", "search.example.com:*"} <= set( + security.allowed_hosts + ) + assert "https://search.example.com:8443" in security.allowed_origins + + def test_extra_hosts_and_origins_are_added(self): + config = _http_config( + host="10.0.0.5", + allowed_hosts=["search.internal:*"], + allowed_origins=["https://portal.example.com"], + ) + + security = _create_server(config, "streamable-http").settings.transport_security + + assert {"search.internal:*", "10.0.0.5:*"} <= set(security.allowed_hosts) + assert "https://portal.example.com" in security.allowed_origins + + def test_public_url_must_be_an_absolute_url_without_path(self): + for bad in ("search.example.com", "https://search.example.com/mcp"): + with pytest.raises(SystemExit): + _create_server(_http_config(public_url=bad), "streamable-http") + + def test_authorization_servers_require_a_public_url(self): + config = _http_config(authorization_servers=["https://idp.example.com/realms/docs"]) + + with pytest.raises(SystemExit): + _create_server(config, "streamable-http") + + def test_proxied_requests_health_and_metadata(self, caplog): + """Through a proxy the Host header is the public hostname; the health + and metadata routes are plain GETs; a forwarded identity is logged.""" + import logging + + from starlette.testclient import TestClient + + config = _http_config( + public_url="https://search.example.com", + authorization_servers=["https://idp.example.com/realms/docs"], + ) + headers = { + "Host": "search.example.com", + "Origin": "https://search.example.com", + "Accept": "application/json, text/event-stream", + } + with patch("chunksilo.search.run_search") as run_search: + run_search.return_value = {"chunks": [], "matched_files": []} + app = _create_server(config, "streamable-http").streamable_http_app() + with TestClient(app) as client: + assert client.post("/mcp", headers=headers, json=_INIT).status_code == 200 + + with caplog.at_level(logging.INFO, logger="chunksilo.server"): + response = client.post( + "/mcp", headers={**headers, "X-Forwarded-User": "alice"}, json=_SEARCH + ) + assert response.status_code == 200 + assert run_search.call_args[0][0] == "widget" + assert "Search by alice" in caplog.text + + assert client.post( + "/mcp", headers={**headers, "Host": "evil.example.net"}, json=_INIT + ).status_code == 421 + + health = client.get("/health") + assert health.status_code == 200 and health.json()["status"] == "ok" + + for path in ("/.well-known/oauth-protected-resource", + "/.well-known/oauth-protected-resource/mcp"): + metadata = client.get(path).json() + assert metadata["resource"] == "https://search.example.com/mcp" + assert metadata["authorization_servers"] == ["https://idp.example.com/realms/docs"] + + def test_metadata_route_is_absent_without_authorization_servers(self): + from starlette.testclient import TestClient + + app = _create_server(_http_config(public_url="https://search.example.com"), "streamable-http").streamable_http_app() + with TestClient(app) as client: + assert client.get("/health").status_code == 200 + assert client.get("/.well-known/oauth-protected-resource").status_code == 404 + + def test_stdio_search_has_no_forwarded_user(self): + """The context parameter must not leak into the tool's input schema.""" + mcp = _create_server() + (tool,) = [t for t in mcp._tool_manager.list_tools() if t.name == "search_docs"] + assert "ctx" not in tool.parameters["properties"] From 98091257063e556b8f57494ff312df8c7488fb76 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Mon, 14 Sep 2026 18:48:41 +0200 Subject: [PATCH 16/18] fix: require mcp 1.10 or newer server.py imports mcp.server.transport_security, which the SDK added in 1.10.0, so chunksilo-mcp failed at import - even over stdio - in an environment that updated the wheel but kept an older mcp: the previous floor of 1.0.0 let pip leave it in place. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index de1666e..331a708 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ llama-index-retrievers-bm25>=0.1.0,<1 pillow>=10.3.0,<11 pypdf>=5.1.0,<7 python-docx>=1.1.0,<2 -mcp>=1.0.0,<2 +mcp>=1.10.0,<2 huggingface-hub>=0.22.0,<2 flashrank>=0.1.0,<1 fastembed>=0.5.0,<1 From 52380bb617d338d17ba954d06a7ca4dee71ee04e Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Mon, 14 Sep 2026 19:15:57 +0200 Subject: [PATCH 17/18] fix: retry loads that failed and keep files under directories the scan could not list Two ways the incremental indexer could quietly lose content: - A loader that hit a timeout, a failed .doc conversion or a file that vanished returned an empty list, which build_index persisted as "holds no text". The file was then never tried again - a .doc indexed before LibreOffice was installed stayed unindexed for ever. Those paths now raise (FileProcessingTimeoutError / FileLoadError), so the file gets no state row and is retried next run, as the LoadOutcome contract says. - os.walk skips a directory it cannot list and carries on, so a subdirectory that lost permissions or a stale mount finished the walk "cleanly" and everything indexed under it was pruned, then re-indexed once readable again. The walk now records such directories on ScanStatus.unreadable_dirs; unscanned_roots() reports them, so deletions under them are withheld like under a stalled root, and --check-files lists them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- AGENTS.md | 4 +- docs/troubleshooting.md | 4 +- src/chunksilo/filecheck.py | 19 ++++++++ src/chunksilo/index.py | 82 ++++++++++++++++++++++++--------- test/test_check_files.py | 20 ++++++++ test/test_incremental_ingest.py | 71 ++++++++++++++++++++++++++++ test/test_scan_timeouts.py | 12 +++-- 7 files changed, 180 insertions(+), 32 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index db56ca9..d270bae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,8 +36,8 @@ test/run_tests.sh live under `storage.storage_dir`; sources are opened read-only and .doc files are copied out before LibreOffice ever sees them. `test/test_never_mutates.py` enforces this with a byte-for-byte snapshot - keep it passing. -- **A file missing from a scan is only deleted if the scan could see where it lives.** `LocalFileSystemSource` records unreadable paths and incomplete walks on `ScanStatus`; `build_index` subtracts those from the deletion set. Pruning on a failed lookup drops the file's chunks and state row, so the next run re-indexes it — on a laggy network mount that becomes a permanent re-index loop. -- **Every attempted file gets a state row unless it failed.** `load_files_parallel` returns a `LoadOutcome` per file: `empty` (extracted no text) is a final answer and is persisted with no doc ids; `failed` gets no row so it is retried. Dropping empty results makes those files look new on every run for ever. +- **A file missing from a scan is only deleted if the scan could see where it lives.** `LocalFileSystemSource` records unreadable files, subdirectories it could not list (`os.walk` skips those silently) and incomplete walks on `ScanStatus`; `build_index` subtracts those from the deletion set. Pruning on a failed lookup drops the file's chunks and state row, so the next run re-indexes it — on a laggy network mount that becomes a permanent re-index loop. +- **Every attempted file gets a state row unless it failed.** `load_files_parallel` returns a `LoadOutcome` per file: `empty` (extracted no text) is a final answer and is persisted with no doc ids; `failed` gets no row so it is retried. Dropping empty results makes those files look new on every run for ever. A loader signals failure by raising (`FileLoadError`, `FileProcessingTimeoutError`, `OSError`), never by returning an empty list: a timeout or a failed `.doc` conversion returned as `[]` would be persisted as "holds no text" and never retried. - **`--check-files` runs the indexer's own scan.** `filecheck.run_check` drives the same `iter_candidates` / `_match_decision` / revision partitioning as `build_index`; a change to the scan that is not visible through those APIs will make the dry run lie. ## Search Invariants diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 60bec4d..4bda9e9 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -10,11 +10,11 @@ Part of [ChunkSilo](../README.md). - **Confluence Integration**: Included by default — set `confluence.url`, `confluence.username`, and `confluence.api_token` in `config.yaml`. - **Jira Integration**: Included by default — set `jira.url`, `jira.username`, and `jira.api_token` in `config.yaml`. Optionally configure `jira.projects` to restrict search to specific project keys. - **Custom CA Bundle**: Set `ssl.ca_bundle_path` in `config.yaml` for custom certificates. -- **Network mounts**: Unavailable directories are skipped with a warning; indexing continues with available directories. Files already indexed from a directory that could not be scanned — or individual files that could not be read — are **kept**, not treated as deleted, and the run reports how many (`N unseen (kept)`). Otherwise a mount that is briefly slow drops its whole tree from the index and the next run re-indexes it from scratch. The consequence is that content genuinely removed from a still-configured directory that has become unreachable stays in the index; to drop it, remove the directory from `indexing.directories` and rebuild. +- **Network mounts**: Unavailable directories are skipped with a warning; indexing continues with available directories. Files already indexed from a directory or subdirectory that could not be scanned — or individual files that could not be read — are **kept**, not treated as deleted, and the run reports how many (`N unseen (kept)`). Otherwise a mount that is briefly slow drops its whole tree from the index and the next run re-indexes it from scratch. The consequence is that content genuinely removed from a still-configured directory that has become unreachable stays in the index; to drop it, remove the directory from `indexing.directories` and rebuild. - **Files that keep being re-indexed**: run `chunksilo --build-index --verbose` and look for `Reprocessing :` — it names the reason (`content changed`, `revision status changed`). Files with no extractable text (an image-only `.docx`, for instance) are recorded as empty and are not retried until they change; files that failed to load are reported separately and *are* retried. - **A file is not being indexed**: run `chunksilo --check-files ` — it names the exact reason (excluded by a glob pattern, no include pattern matched, pruned directory, superseded revision, review copy, disabled or unavailable directory) without re-indexing, and `chunksilo --check-files` alone reports a verdict for every file the scan sees (see [cli.md](cli.md#testing-the-file-filters---check-files)). By default only the newest revision of a document group is indexed, and review-comment copies never are; during a real build the scan summary counts them (`N superseded (skipped)`, `N review copies (skipped)`) and `--verbose` names each file. If the classification is wrong, adjust `indexing.versioning.review_copy_patterns` / `doc_id_patterns`, set `indexing.versioning.index_superseded: true` to keep old revisions searchable, or set `indexing.versioning.enabled: false` to turn revision handling off entirely (see [configuration.md](configuration.md)). - **Results all come from one document**: `retrieval.max_chunks_per_doc` (default 2) caps how many chunks of one document appear; raise `retrieval.embed_top_k` to widen the pool it chooses from. -- **Legacy .doc files**: Requires LibreOffice to be installed for automatic conversion to .docx. If LibreOffice is not found, .doc files are skipped with a warning. Full heading extraction is supported. Conversion runs on a copy of the file in a private directory under `storage.storage_dir`, so nothing is ever written next to the original. +- **Legacy .doc files**: Requires LibreOffice to be installed for automatic conversion to .docx. If LibreOffice is not found, each .doc file is reported as failed and tried again on every run until it is installed. Full heading extraction is supported. Conversion runs on a copy of the file in a private directory under `storage.storage_dir`, so nothing is ever written next to the original. - **Old config options**: after upgrading from 2.x, a warning such as `'indexing.timeout' is no longer used and is ignored; use ...` means the file still sets a removed option; see the [upgrade notes](configuration.md#upgrading-from-2x). - **HTTP transport**: `chunksilo-mcp --transport streamable-http` binds `127.0.0.1:8400` by default and authenticates nobody. If a remote client cannot connect, that is the loopback bind doing its job — put an authenticating reverse proxy in front (see [reverse-proxy.md](reverse-proxy.md)) rather than exposing the port directly. - **`421 Invalid Host header` behind a proxy**: the MCP transport only accepts the `Host` values it expects, and a proxy forwards the public hostname. Set `server.public_url` to the URL the proxy serves (add `server.allowed_hosts` for any further names). A `403 Invalid Origin header` is the same check for browser-based clients: `server.allowed_origins`. diff --git a/src/chunksilo/filecheck.py b/src/chunksilo/filecheck.py index e0763ba..61cc08e 100644 --- a/src/chunksilo/filecheck.py +++ b/src/chunksilo/filecheck.py @@ -64,6 +64,7 @@ class DirectoryReport: scan_complete: bool = True scan_incomplete_reason: str | None = None pruned_dirs: list[PrunedDir] = field(default_factory=list) + unreadable_dirs: list[str] = field(default_factory=list) # absolute files: list[FileVerdict] = field(default_factory=list) counts: dict[str, int] = field(default_factory=dict) @@ -170,6 +171,8 @@ def run_check(config: dict[str, Any], target: str | None = None) -> CheckReport: reason="could not stat (error or timeout); " "would be skipped and its deletion withheld", )) + elif event.kind == "unreadable_dir": + dir_report.unreadable_dirs.append(event.path) # Revision partitioning across ALL sources at once, exactly like # build_index - doc-ID groups can span directories. @@ -230,13 +233,16 @@ def run_check(config: dict[str, Any], target: str | None = None) -> CheckReport: for dir_report in report.directories: dir_report.files.sort(key=lambda v: v.path) dir_report.pruned_dirs.sort(key=lambda p: p.path) + dir_report.unreadable_dirs.sort() dir_report.counts = {status: 0 for status in STATUSES} for verdict in dir_report.files: dir_report.counts[verdict.status] += 1 dir_report.counts["pruned_dirs"] = len(dir_report.pruned_dirs) + dir_report.counts["unreadable_dirs"] = len(dir_report.unreadable_dirs) report.totals = {status: 0 for status in STATUSES} report.totals["pruned_dirs"] = 0 + report.totals["unreadable_dirs"] = 0 for dir_report in report.directories: for key, count in dir_report.counts.items(): report.totals[key] += count @@ -279,6 +285,14 @@ def _resolve_target(target, report, data_source): f"by pattern {pruned.pattern!r}", pattern=pruned.pattern, ) + for unreadable_dir in dir_report.unreadable_dirs: + if _under(path, unreadable_dir): + return FileVerdict( + path=path, + status="unreadable", + reason=f"ancestor directory {unreadable_dir} could not be " + "listed; files under it are kept, not re-indexed", + ) for dir_report in report.directories: if not _under(path, dir_report.path): @@ -386,6 +400,10 @@ def render_text(report: CheckReport, out: TextIO | None = None) -> None: file=out) for pruned in dir_report.pruned_dirs: print(f" pruned dir {pruned.path} [by {pruned.pattern!r}]", file=out) + for unreadable_dir in dir_report.unreadable_dirs: + print(f" WARNING: could not list {unreadable_dir}; files under it are " + "missing from this report and their deletion is withheld", + file=out) for verdict in dir_report.files: label = _STATUS_LABELS[verdict.status].ljust(13) suffix = "" @@ -435,6 +453,7 @@ def _summary_line(totals: dict[str, int]) -> str: "unreadable": "unreadable", "duplicate": "duplicates", "pruned_dirs": "directories pruned", + "unreadable_dirs": "directories unlistable", } for key, name in names.items(): if totals.get(key): diff --git a/src/chunksilo/index.py b/src/chunksilo/index.py index 2950224..d05e5af 100644 --- a/src/chunksilo/index.py +++ b/src/chunksilo/index.py @@ -306,7 +306,7 @@ class FileCandidate: class ScanEvent: """A file or directory the scan saw but will not index. Diagnostic only.""" - kind: str # "excluded_file" | "pruned_dir" | "unreadable_file" + kind: str # "excluded_file" | "pruned_dir" | "unreadable_file" | "unreadable_dir" path: str # absolute pattern: str | None = None # deciding pattern, when one exists @@ -326,6 +326,10 @@ class ScanStatus: complete: bool = True reason: str = "" unreadable: set[str] = field(default_factory=set) + # Subdirectories os.walk could not list -> reason. The walk carries on + # past them, so the scan as a whole is still complete; only what lies + # under these is withheld from deletion. + unreadable_dirs: dict[str, str] = field(default_factory=dict) def fail(self, reason: str) -> None: self.complete = False @@ -468,6 +472,11 @@ def load_file( ) -> list[LlamaIndexDocument]: """Load and return documents for a given file. + An empty list means the file was read and holds no text. A file that + could not be read or converted must raise (FileLoadError, + FileProcessingTimeoutError, OSError, ...) rather than return [] - the + caller then records no state for it and retries it next run. + Args: file_info: File information ctx: Optional processing context for progress updates and timeout @@ -643,6 +652,14 @@ def flatten_outline(items, level=1): _SCAN_TIMEOUT_SENTINEL = object() +class FileLoadError(Exception): + """A file could not be read or converted. + + Distinct from a file that loads cleanly and holds no text: a load that + raises gets no state row, so the next run tries the file again. + """ + + def _run_with_timeout(fn, timeout_seconds: float, default=_SCAN_TIMEOUT_SENTINEL): """Run *fn* in a background thread, returning *default* on timeout. @@ -797,14 +814,31 @@ def _walk_with_timeout( on_event, when given, receives a ScanEvent per pruned directory. It may be invoked from the walker thread. + + A directory os.walk cannot list is recorded on + scan_status.unreadable_dirs - os.walk itself would skip it silently + and finish as if the tree were complete - or fails the scan when it + is the root itself. """ timeout = cfgload.get("indexing.scan_item_seconds") q: queue.Queue = queue.Queue() _sentinel = None # signals end of iteration + base_abs = str(self.base_dir.absolute()) + + def _on_error(exc: OSError) -> None: + failed = str(Path(exc.filename).absolute()) if exc.filename else base_abs + logger.warning(f"Could not list directory {failed}: {exc}") + if failed == base_abs: + self.scan_status.fail(f"could not be listed: {exc}") + else: + self.scan_status.unreadable_dirs[failed] = f"could not be listed: {exc}" + if on_event is not None: + on_event(ScanEvent("unreadable_dir", failed)) + def _producer(): try: - for root, dirs, files in os.walk(self.base_dir): + for root, dirs, files in os.walk(self.base_dir, onerror=_on_error): kept_dirs = [] for d in dirs: skip_pattern = self._directory_skip_pattern(d) @@ -1020,9 +1054,11 @@ def _create_file_info_inner( ) def unscanned_roots(self) -> dict[str, str]: - if self.scan_status.complete: - return {} - return {str(self.base_dir.absolute()): self.scan_status.reason} + roots: dict[str, str] = {} + if not self.scan_status.complete: + roots[str(self.base_dir.absolute())] = self.scan_status.reason + roots.update(self.scan_status.unreadable_dirs) + return roots def unreadable_paths(self) -> set[str]: return set(self.scan_status.unreadable) @@ -1035,12 +1071,14 @@ def load_file( file_path = Path(file_info.path) exists_timeout = cfgload.get("indexing.scan_item_seconds") exists_result = _run_with_timeout( - file_path.exists, timeout_seconds=exists_timeout, default=False, + file_path.exists, timeout_seconds=exists_timeout, ) + if exists_result is _SCAN_TIMEOUT_SENTINEL: + raise FileProcessingTimeoutError( + f"existence check timed out after {exists_timeout}s" + ) if not exists_result: - if exists_result is False: - logger.warning(f"Skipping disappeared file: {file_path}") - return [] + raise FileLoadError("file disappeared before it could be read") if file_path.suffix.lower() == ".docx": if ctx: ctx.set_phase("Parsing DOCX") @@ -1057,10 +1095,9 @@ def load_file( default=None, ) if result is None: - logger.warning( - f"DOCX processing timed out after {remaining:.0f}s: {file_path}" + raise FileProcessingTimeoutError( + f"DOCX processing timed out after {remaining:.0f}s" ) - return [] return result return split_docx_into_heading_documents( file_path, ctx, @@ -1078,8 +1115,7 @@ def load_file( ) if docx_path is None: - logger.warning(f"Skipping {file_path}: could not convert .doc to .docx") - return [] + raise FileLoadError("could not convert .doc to .docx") try: if ctx: ctx.set_phase("Parsing converted DOCX") @@ -1097,10 +1133,10 @@ def load_file( default=None, ) if result is None: - logger.warning( - f"DOCX processing timed out after {remaining:.0f}s: {docx_path}" + raise FileProcessingTimeoutError( + f"DOCX processing timed out after {remaining:.0f}s" ) - docs = result if result is not None else [] + docs = result else: docs = split_docx_into_heading_documents( docx_path, ctx, @@ -1132,10 +1168,10 @@ def load_file( default=None, ) if result is None: - logger.warning( + raise FileProcessingTimeoutError( f"load_data() timed out after {remaining:.0f}s" ) - docs = result if result is not None else [] + docs = result else: docs = reader.load_data() # Ensure dates are visible to LLM (remove from exclusion list) @@ -1244,13 +1280,13 @@ def unscanned_roots(self) -> dict[str, str]: """Directory roots that were not fully enumerated, mapped to the reason. A root lands here when it was unavailable at startup or when its walk - stalled or could not be listed. Callers must not prune tracked files - under these roots. + stalled or could not be listed; a subdirectory lands here when the walk + could not list it. Callers must not prune tracked files under any of + them. """ roots = dict(self._unavailable_reasons) for source in self.sources: - if not source.scan_status.complete: - roots[str(source.base_dir.absolute())] = source.scan_status.reason + roots.update(source.unscanned_roots()) return roots def unreadable_paths(self) -> set[str]: diff --git a/test/test_check_files.py b/test/test_check_files.py index 8cf0a9d..18b7525 100644 --- a/test/test_check_files.py +++ b/test/test_check_files.py @@ -196,6 +196,26 @@ def test_unreadable_file_event(self, tmp_path): assert [e.kind for e in events] == ["unreadable_file"] assert events[0].path == str((tmp_path / "a.pdf").absolute()) + @pytest.mark.skipif(os.geteuid() == 0, reason="root can list a directory without read permission") + def test_unreadable_dir_event(self, tmp_path): + sub = tmp_path / "locked" + sub.mkdir() + (sub / "a.pdf").write_text("x") + source = _make_source(tmp_path, include=["**/*.pdf"], exclude=[]) + events = [] + sub.chmod(0) + try: + candidates = list(source.iter_candidates(on_event=events.append)) + finally: + sub.chmod(0o755) + assert candidates == [] + assert [e.kind for e in events] == ["unreadable_dir"] + assert events[0].path == str(sub.absolute()) + # The rest of the tree was walked: the scan is complete, only this + # subtree is withheld from deletion. + assert source.scan_status.complete + assert list(source.unscanned_roots()) == [str(sub.absolute())] + def test_non_recursive_events(self, tmp_path): (tmp_path / "a.pdf").write_text("x") (tmp_path / "b.xlsx").write_text("x") diff --git a/test/test_incremental_ingest.py b/test/test_incremental_ingest.py index b03ac74..8d03514 100644 --- a/test/test_incremental_ingest.py +++ b/test/test_incremental_ingest.py @@ -326,3 +326,74 @@ def test_reprocess_reason_is_logged(test_env): build_index() assert "Reprocessing" in log.text assert "content changed" in log.text + + +def test_doc_conversion_failure_is_retried(test_env): + """A .doc that could not be converted gets no state row, so it is tried + again next run (for instance once LibreOffice is installed). Persisting it + as a file that holds no text would silence it for ever.""" + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "legacy.doc", "not really a doc") + + with patch("chunksilo.index._convert_doc_to_docx", return_value=None), \ + capture_index_log() as log: + build_index() + assert str((data_dir / "legacy.doc").absolute()) not in _tracked(db_path) + assert "could not convert" in log.text + + with patch("chunksilo.index._convert_doc_to_docx", return_value=None) as convert: + build_index() + assert convert.called, "the failed .doc must be attempted again" + + +def test_timed_out_load_is_retried(test_env): + """A file whose reader timed out gets no state row, so it is retried.""" + data_dir, _storage_dir, db_path = test_env + create_file(data_dir, "slow.txt", "some text") + real_run = index._run_with_timeout + + def timing_out(fn, timeout_seconds, default=index._SCAN_TIMEOUT_SENTINEL): + if getattr(fn, "__name__", "") == "load_data": + return default # exactly what a timeout returns + return real_run(fn, timeout_seconds, default) + + with patch.object(index, "_run_with_timeout", timing_out), capture_index_log() as log: + build_index() + assert str((data_dir / "slow.txt").absolute()) not in _tracked(db_path) + assert "timed out" in log.text + + with capture_index_log() as log: + build_index() + assert "Indexing new file" in log.text + assert str((data_dir / "slow.txt").absolute()) in _tracked(db_path) + + +@pytest.mark.skipif(os.geteuid() == 0, reason="root can list a directory without read permission") +def test_unlistable_subdirectory_does_not_delete_its_files(test_env): + """A subdirectory the walk cannot list must not have its files pruned. + + os.walk skips a directory it cannot open and carries on, so without an + error callback the walk ends "cleanly" and everything indexed under that + directory looks deleted - and is re-indexed once it is readable again. + """ + data_dir, _storage_dir, db_path = test_env + sub = data_dir / "sub" + sub.mkdir() + create_file(data_dir, "doc1.txt", "This is document 1.") + create_file(sub, "doc2.txt", "This is document 2.") + build_index() + assert len(_tracked(db_path)) == 2 + + sub.chmod(0) + try: + with capture_index_log() as log: + build_index() + finally: + sub.chmod(0o755) + assert len(_tracked(db_path)) == 2, "the directory was unlistable, not emptied" + assert "did not see" in log.text + + with capture_index_log() as log: + build_index() + assert len(_tracked(db_path)) == 2 + assert "Indexing new file" not in log.text diff --git a/test/test_scan_timeouts.py b/test/test_scan_timeouts.py index d87dbb6..4163ad0 100644 --- a/test/test_scan_timeouts.py +++ b/test/test_scan_timeouts.py @@ -17,6 +17,8 @@ import pytest +from chunksilo.ui import FileProcessingTimeoutError + from chunksilo.index import ( _run_with_timeout, _SCAN_TIMEOUT_SENTINEL, @@ -339,9 +341,9 @@ def _stalling_walk(top, **kw): class TestLoadFileExistsTimeout: @patch("chunksilo.index.cfgload") - def test_exists_hang_skips_file(self, mock_cfg, tmp_path): - """When Path.exists blocks on a file, load_file returns [] rather - than hanging.""" + def test_exists_hang_fails_file(self, mock_cfg, tmp_path): + """When Path.exists blocks on a file, load_file raises a timeout - so + the file gets no state row and is retried - rather than hanging.""" mock_cfg.get.return_value = 2 # 2s timeout f = tmp_path / "test.txt" @@ -362,8 +364,8 @@ def _blocking_exists(self_path): with patch.object(Path, "exists", _blocking_exists): start = time.monotonic() - result = source.load_file(fi) + with pytest.raises(FileProcessingTimeoutError): + source.load_file(fi) elapsed = time.monotonic() - start - assert result == [] assert elapsed < 10, f"load_file() took {elapsed:.1f}s, expected < 10s" From bde6b92d81bc10ff95889c64b8e824a1b3b53363 Mon Sep 17 00:00:00 2001 From: Fredrik Reveny Date: Mon, 14 Sep 2026 19:15:57 +0200 Subject: [PATCH 18/18] fix: absolutise legacy relative result paths before building file URIs Metadata written by old index versions can hold a relative path. Joined to a relative configured directory such as ./data it stayed relative, and file://data/x.pdf reads "data" as a host. os.path.abspath is string arithmetic against the CWD, so the no-filesystem rule still holds. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011Yp9BfFC7MRYZ4WCKSobE6 --- src/chunksilo/search.py | 6 +++++- test/test_chunk_location.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/chunksilo/search.py b/src/chunksilo/search.py index 7a8af5e..638dd1f 100644 --- a/src/chunksilo/search.py +++ b/src/chunksilo/search.py @@ -169,7 +169,11 @@ def _normalize_result_path(file_path: str, config: dict[str, Any]) -> str | None # than probing the filesystem for the one that happens to hold them. directories = _get_configured_directories(config) base = str(directories[0]) if directories else "" - absolute = os.path.join(base, raw) if base else raw + joined = os.path.join(base, raw) if base else raw + # A relative configured directory ("./data") would otherwise leave the + # result relative, and file://data/... reads "data" as a host. abspath + # is string arithmetic against the CWD: no stat, no readlink. + absolute = os.path.abspath(joined) return os.path.normpath(absolute) diff --git a/test/test_chunk_location.py b/test/test_chunk_location.py index 340165c..aace8ad 100644 --- a/test/test_chunk_location.py +++ b/test/test_chunk_location.py @@ -215,6 +215,16 @@ def test_relative_file_path(self): == "file:///Users/test/data/docs/readme.md" ) + def test_relative_path_under_relative_directory(self, monkeypatch): + """With a relative configured directory the result must still be + absolute: file://data/x.pdf would read "data" as a host.""" + monkeypatch.setattr(os, "getcwd", lambda: "/srv/chunksilo") + config = {"indexing": {"directories": ["./data"]}} + assert ( + _resolve_file_uri("docs/readme.md", config) + == "file:///srv/chunksilo/data/docs/readme.md" + ) + def test_no_filesystem_access(self, monkeypatch): """Nothing in URI building may stat, readlink or resolve.""" def explode(*_args, **_kwargs):