mod a nits: finish Module A handoff + OIE orchestrator

.gitignore

@@ -73,6 +73,7 @@ standards_cache.sqlite
 !docs/Final_eval_blog_gsoc2026/module_B_final_blog.md
 !application/utils/librarian/README.md
 !docs/gsoc_2026_module_c/*.md
+!docs/gsoc_2026_module_a/*.md
 !docs/gsoc_2026_module_b/module_a_contract.md
 !docs/gsoc_2026_module_b/module_c_contract.md
 !docs/gsoc_2026_module_b/module_b_runbook.md

Makefile

@@ -181,6 +181,14 @@ alembic-guardrail:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	python scripts/check_alembic_revision_guardrail.py
 
+oie-pipeline:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS)
+
+oie-e2e-smoke:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/run_oie_e2e_smoke.py
+
 openapi-generate:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	python scripts/generate_openapi.py

application/cmd/cre_main.py

@@ -892,6 +892,25 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
         logger.info("Exported %s rows to %s", rows, csv_out)
         return
 
+    if getattr(args, "run_harvester", False):
+        import sys
+
+        from application import sqla
+        from application.utils.harvester.pipeline import run_harvester
+
+        db_connect(args.cache_file)
+        repos_yaml = getattr(args, "harvester_repos_yaml", "") or None
+        summary = run_harvester(
+            sqla.session,
+            args.run_id.strip(),
+            repos_yaml=repos_yaml,
+            dry_run=getattr(args, "harvester_dry_run", False),
+        )
+        print(summary.to_json())
+        if summary.status == "degraded":
+            sys.exit(1)
+        return
+
     if getattr(args, "run_noise_filter", False):
         import sys
 

application/tests/harvester_test/artifact_registry_test.py

@@ -0,0 +1,99 @@
+import unittest
+from datetime import datetime
+
+from application.utils.harvester.artifact_registry import ArtifactRegistry
+from application.utils.harvester.models import ArtifactRegistryRecord
+
+
+class ArtifactRegistryTests(unittest.TestCase):
+    def test_insert_record(self):
+        registry = ArtifactRegistry()
+
+        record = ArtifactRegistryRecord(
+            artifact_id="art:test:file.md",
+            repository="OWASP/ASVS",
+            locator_path="file.md",
+            content_hash="abc",
+            last_commit_sha="123",
+            last_pipeline_run="run1",
+            last_processed_at=datetime.now(),
+            status="new",
+        )
+
+        registry.upsert(record)
+
+        self.assertTrue(registry.exists(record.artifact_id))
+
+    def test_get_record(self):
+        registry = ArtifactRegistry()
+
+        record = ArtifactRegistryRecord(
+            artifact_id="art:test:file.md",
+            repository="OWASP/ASVS",
+            locator_path="file.md",
+            content_hash="abc",
+            last_commit_sha="123",
+            last_pipeline_run="run1",
+            last_processed_at=datetime.now(),
+            status="new",
+        )
+
+        registry.upsert(record)
+
+        stored = registry.get(record.artifact_id)
+        assert stored is not None
+
+        self.assertEqual(stored.content_hash, "abc")
+
+    def test_update_record(self):
+        registry = ArtifactRegistry()
+
+        record = ArtifactRegistryRecord(
+            artifact_id="art:test:file.md",
+            repository="OWASP/ASVS",
+            locator_path="file.md",
+            content_hash="abc",
+            last_commit_sha="123",
+            last_pipeline_run="run1",
+            last_processed_at=datetime.now(),
+            status="new",
+        )
+
+        registry.upsert(record)
+
+        record.content_hash = "xyz"
+        record.status = "updated"
+
+        registry.upsert(record)
+
+        stored = registry.get(record.artifact_id)
+        assert stored is not None
+
+        self.assertEqual(stored.content_hash, "xyz")
+        self.assertEqual(stored.status, "updated")
+
+    def test_all_records(self):
+        registry = ArtifactRegistry()
+
+        for i in range(3):
+            registry.upsert(
+                ArtifactRegistryRecord(
+                    artifact_id=f"art:{i}",
+                    repository="repo",
+                    locator_path=f"{i}.md",
+                    content_hash=str(i),
+                    last_commit_sha="sha",
+                    last_pipeline_run="run",
+                    last_processed_at=datetime.now(),
+                    status="new",
+                )
+            )
+
+        self.assertEqual(
+            len(registry.all()),
+            3,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/checkpoint_manager_test.py

@@ -0,0 +1,68 @@
+import unittest
+from datetime import datetime
+
+from application.utils.harvester.checkpoint_manager import CheckpointManager
+from application.utils.harvester.models import CheckpointRecord
+
+
+class CheckpointManagerTests(unittest.TestCase):
+    def test_save_checkpoint(self):
+        manager = CheckpointManager()
+
+        checkpoint = CheckpointRecord(
+            repository="OWASP/ASVS",
+            pipeline_run_id="run1",
+            last_processed_commit="abc123",
+            status="running",
+            updated_at=datetime.now(),
+        )
+
+        manager.save(checkpoint)
+
+        self.assertIsNotNone(manager.get("OWASP/ASVS"))
+
+    def test_update_commit(self):
+        manager = CheckpointManager()
+
+        checkpoint = CheckpointRecord(
+            repository="OWASP/ASVS",
+            pipeline_run_id="run1",
+            last_processed_commit="abc123",
+            status="running",
+            updated_at=datetime.now(),
+        )
+
+        manager.save(checkpoint)
+
+        manager.update_commit(
+            "OWASP/ASVS",
+            "deadbeef",
+        )
+
+        stored = manager.get("OWASP/ASVS")
+        assert stored is not None
+
+        self.assertEqual(stored.last_processed_commit, "deadbeef")
+
+    def test_mark_completed(self):
+        manager = CheckpointManager()
+
+        checkpoint = CheckpointRecord(
+            repository="OWASP/ASVS",
+            pipeline_run_id="run1",
+            last_processed_commit="abc123",
+            status="running",
+            updated_at=datetime.now(),
+        )
+
+        manager.save(checkpoint)
+        manager.mark_completed("OWASP/ASVS")
+
+        stored = manager.get("OWASP/ASVS")
+        assert stored is not None
+
+        self.assertEqual(stored.status, "completed")
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/chunk_pipeline_test.py

@@ -0,0 +1,36 @@
+import unittest
+from unittest.mock import Mock
+
+from application.utils.harvester.chunk_pipeline import DocumentChunkPipeline
+from application.utils.harvester.models import Document, IngestChunkRecord
+
+
+class DocumentChunkPipelineTests(unittest.TestCase):
+    def test_invalid_record_is_rejected_before_return(self):
+        document = Mock(spec=Document)
+        document.text = "Some document text."
+
+        chunker = Mock()
+        chunker.chunk.return_value = ["chunk"]
+
+        record_builder = Mock()
+        invalid_record = Mock(spec=IngestChunkRecord)
+        record_builder.build.return_value = [invalid_record]
+
+        validator = Mock()
+        validator.validate.side_effect = ValueError("invalid chunk record")
+
+        pipeline = DocumentChunkPipeline(
+            chunker=chunker,
+            record_builder=record_builder,
+            validator=validator,
+        )
+
+        with self.assertRaisesRegex(ValueError, "invalid chunk record"):
+            pipeline.chunk(document)
+
+        validator.validate.assert_called_once_with(invalid_record)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/chunk_record_builder_test.py

@@ -0,0 +1,75 @@
+import unittest
+from datetime import datetime, timezone
+
+from application.utils.harvester.chunk_record_builder import ChunkRecordBuilder
+from application.utils.harvester.chunk_record_validator import (
+    ChunkRecordValidator,
+    ingest_record_to_payload,
+)
+from application.utils.harvester.chunker import ChunkInfo
+from application.utils.harvester.models import (
+    Document,
+    HeadingNode,
+    Locator,
+    SourceInfo,
+)
+from application.utils.noise_filter.schemas import ChangeRecord
+
+
+class ChunkRecordBuilderTests(unittest.TestCase):
+    def _document(self, text: str, headings: list[HeadingNode]) -> Document:
+        return Document(
+            schema_version="0.2.0",
+            artifact_id="art:OWASP/ASVS:README.md",
+            pipeline_run_id="run-1",
+            text=text,
+            source=SourceInfo(
+                type="github",
+                repository="OWASP/ASVS",
+                commit_sha="abc1234deadbeef",
+                committed_at=datetime(2026, 2, 1, 1, 0, 0, tzinfo=timezone.utc),
+            ),
+            locator=Locator(
+                kind="repo_path",
+                id="README.md",
+                path="README.md",
+            ),
+            heading_structure=headings,
+        )
+
+    def test_builds_change_record_shaped_payload(self) -> None:
+        text = "# Root\n\nFirst paragraph."
+        document = self._document(
+            text,
+            [HeadingNode(level=1, text="Root", start_line=1, end_line=3)],
+        )
+        chunk = ChunkInfo(text=text, start_char_idx=0, end_char_idx=len(text))
+        records = ChunkRecordBuilder().build(document, [chunk])
+        self.assertEqual(len(records), 1)
+        record = records[0]
+        self.assertEqual(record.chunk_id, "chk:art:OWASP/ASVS:README.md:0")
+        self.assertEqual(record.pipeline_run_id, "run-1")
+        self.assertEqual(record.source_repo, "OWASP/ASVS")
+        self.assertEqual(record.locator_path, "README.md")
+        self.assertEqual(record.span.heading_path, ["Root"])
+
+        ChunkRecordValidator().validate(record)
+        payload = ingest_record_to_payload(record)
+        ChangeRecord.model_validate(payload)
+
+    def test_indexes_multiple_chunks(self) -> None:
+        text = "AAAA\n\nBBBB"
+        document = self._document(text, [])
+        chunks = [
+            ChunkInfo(text="AAAA\n\n", start_char_idx=0, end_char_idx=6),
+            ChunkInfo(text="BBBB", start_char_idx=6, end_char_idx=10),
+        ]
+        records = ChunkRecordBuilder().build(document, chunks)
+        self.assertEqual(records[0].span.index, 0)
+        self.assertEqual(records[0].span.total, 2)
+        self.assertEqual(records[1].span.index, 1)
+        self.assertEqual(records[1].chunk_id, "chk:art:OWASP/ASVS:README.md:1")
+
+
+if __name__ == "__main__":
+    unittest.main()