From 981b292d8f9f063bd945c29b3bc578d14532c1aa Mon Sep 17 00:00:00 2001 From: rostislav Date: Fri, 7 Aug 2026 16:56:14 +0300 Subject: [PATCH 1/6] add readable multi-branch Qdrant aliases - publish atomic aliases for primary and retained branch generations - preserve immutable generation aliases as the analysis source of truth - reconcile aliases for existing active indexes after deployment - serialize updates per branch while allowing different branches to run in parallel - retain backward-compatible project-level aliases - update RAG configuration and operator documentation - add Java and Python coverage for alias publication and reconciliation --- .../java-shared/application.properties.sample | 5 + .../analysisapi/rag/RagOperationsService.java | 22 +- .../aiclient/AiAnalysisClient.java | 24 + .../analysis/BranchAnalysisProcessor.java | 21 +- .../PullRequestAnalysisProcessor.java | 103 ++- .../service/BranchArchiveService.java | 130 +++- .../service/branch/BranchHealthService.java | 23 +- .../aiclient/AiAnalysisClientTest.java | 41 ++ .../analysis/BranchAnalysisProcessorTest.java | 32 + .../service/BranchArchiveServiceTest.java | 32 + java-ecosystem/libs/commit-graph/pom.xml | 5 + .../src/main/java/module-info.java | 1 + .../commitgraph/model/AnalyzedCommit.java | 21 +- .../persistence/AnalyzedCommitRepository.java | 14 + .../service/AnalyzedCommitService.java | 63 +- .../service/BranchCommitService.java | 65 +- .../service/CommitCoverageService.java | 21 +- .../service/BranchCommitServiceTest.java | 10 +- .../service/CommitCoverageServiceTest.java | 19 + .../core/dto/analysis/issue/IssueDTO.java | 44 +- .../codecrow/core/dto/project/ProjectDTO.java | 21 +- .../core/model/branch/BranchIssue.java | 25 + .../model/codeanalysis/CodeAnalysisIssue.java | 21 + .../model/project/config/ProjectConfig.java | 4 +- .../core/model/project/config/RagConfig.java | 66 +- .../core/model/rag/RagBranchIndex.java | 102 +++ .../model/rag/RagBranchIndexGeneration.java | 149 +++++ .../rag/RagBranchIndexGenerationStatus.java | 10 + .../core/model/rag/RagBranchIndexKind.java | 10 + .../rag/RagBranchIndexLifecycleStatus.java | 10 + .../core/model/rag/RagIndexOperation.java | 137 ++++ .../model/rag/RagIndexOperationStatus.java | 10 + .../RagBranchIndexGenerationRepository.java | 32 + .../rag/RagBranchIndexRepository.java | 3 + .../rag/RagIndexOperationRepository.java | 20 + .../core/service/CodeAnalysisService.java | 4 + .../V2.23.0__exact_branch_index_registry.sql | 126 ++++ ...scm_evidence_and_branch_analysis_scope.sql | 76 +++ .../model/project/config/RagConfigTest.java | 63 ++ .../rag/RagBranchIndexGenerationTest.java | 78 +++ ...BranchIndexBuildExecutorConfiguration.java | 28 + .../BranchIndexGenerationBuildService.java | 245 +++++++ .../branch/BranchIndexMaintenanceService.java | 260 ++++++++ ...nchOperatorAliasReconciliationService.java | 74 +++ .../RagIndexOperationRecoveryService.java | 52 ++ ...RagTransientBranchIndexCleanupService.java | 77 +++ .../ragengine/client/RagPipelineClient.java | 283 +++++++- .../service/IncrementalRagUpdateService.java | 113 +++- .../RagBranchIndexRegistryService.java | 264 ++++++++ .../service/RagOperationsServiceImpl.java | 310 ++++++++- .../service/VcsRagIndexingService.java | 4 + .../source/RepositorySourceTreeIdentity.java | 183 ++++++ ...BranchIndexGenerationBuildServiceTest.java | 174 +++++ ...peratorAliasReconciliationServiceTest.java | 59 ++ .../RagIndexOperationRecoveryServiceTest.java | 31 + ...ransientBranchIndexCleanupServiceTest.java | 59 ++ .../client/RagPipelineClientTest.java | 84 ++- .../IncrementalRagUpdateServiceTest.java | 40 ++ .../RagBranchIndexRegistryServiceTest.java | 179 ++++++ .../service/RagOperationsServiceImplTest.java | 110 +++- .../RepositorySourceTreeIdentityTest.java | 50 ++ java-ecosystem/libs/scm-evidence/pom.xml | 43 ++ .../src/main/java/module-info.java | 25 + .../scmevidence/api/AnalysisReceiptView.java | 11 + .../scmevidence/api/CommitEvidenceView.java | 8 + .../scmevidence/api/IssueProvenance.java | 10 + .../scmevidence/api/PromotionPlan.java | 16 + .../config/ScmEvidenceAutoConfiguration.java | 18 + .../model/ScmAddedLineEvidence.java | 41 ++ .../scmevidence/model/ScmAnalysisReceipt.java | 55 ++ .../scmevidence/model/ScmCommitEvidence.java | 56 ++ .../ScmAddedLineEvidenceRepository.java | 25 + .../ScmAnalysisReceiptRepository.java | 14 + .../ScmCommitEvidenceRepository.java | 17 + .../scmevidence/service/PatchIdentity.java | 51 ++ .../service/ScmEvidenceService.java | 253 ++++++++ .../service/ScmPromotionPlanner.java | 46 ++ .../service/UnifiedDiffAddedLineParser.java | 43 ++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + .../ScmEvidenceAutoConfigurationTest.java | 33 + .../service/ScmEvidenceServiceTest.java | 142 ++++ .../service/ScmPromotionPlannerTest.java | 109 ++++ .../UnifiedDiffAddedLineParserTest.java | 34 + java-ecosystem/pom.xml | 7 + .../controller/RagIndexingController.java | 32 +- .../project/controller/ProjectController.java | 25 +- .../dto/request/UpdateRagConfigRequest.java | 32 +- .../dto/response/RagBranchIndexStatusDTO.java | 19 + .../project/service/IProjectService.java | 5 + .../project/service/ProjectService.java | 27 +- .../service/RagBranchIndexStatusService.java | 127 ++++ .../service/RagIndexingTriggerService.java | 19 +- .../RagBranchIndexStatusServiceTest.java | 56 ++ .../inference-orchestrator/src/model/dtos.py | 40 ++ .../src/service/rag/rag_client.py | 132 +++- .../review/orchestrator/orchestrator.py | 77 ++- .../orchestrator/stage_1_file_review.py | 167 ++++- .../review/orchestrator/stage_2_cross_file.py | 42 +- .../src/service/review/prompt_dry_run.py | 23 +- .../src/service/review/quality_capture.py | 268 +++++++- .../src/service/review/review_service.py | 22 +- .../tests/prompt_dry_run_neutral_fixture.py | 27 +- .../tests/test_candidate_ledger.py | 14 + .../tests/test_prompt_dry_run.py | 49 ++ .../tests/test_quality_capture.py | 121 +++- .../tests/test_rag_client.py | 16 + .../tests/test_rag_client_duplication_unit.py | 23 +- .../tests/test_review_service_helpers.py | 25 +- .../tests/test_stage_1_file_review.py | 224 +++++++ .../tests/test_stage_2_helpers.py | 34 + .../rag-pipeline/requirements.local.txt | 6 +- .../rag-pipeline/requirements.txt | 6 +- .../src/rag_pipeline/api/models.py | 128 ++++ .../src/rag_pipeline/api/routers/index.py | 171 ++++- .../src/rag_pipeline/api/routers/inspect.py | 7 + .../src/rag_pipeline/api/routers/pr.py | 189 +++++- .../src/rag_pipeline/api/routers/query.py | 327 +++++++++- .../src/rag_pipeline/core/coordination.py | 51 +- .../rag_pipeline/core/generation_manifest.py | 379 +++++++++++ .../core/index_manager/branch_manager.py | 17 +- .../core/index_manager/collection_manager.py | 83 ++- .../core/index_manager/indexer.py | 351 ++++++++-- .../core/index_manager/manager.py | 570 ++++++++++++++++- .../core/index_manager/point_operations.py | 52 +- .../rag_pipeline/core/index_representation.py | 4 + .../src/rag_pipeline/core/loader.py | 196 +++--- .../rag_pipeline/core/pr_overlay_identity.py | 5 + .../rag_pipeline/core/pr_overlay_manifest.py | 374 +++++++++++ .../core/pr_overlay_representation.py | 9 + .../src/rag_pipeline/core/revision_binding.py | 78 +++ .../rag_pipeline/core/revision_preflight.py | 478 ++++++++++++++ .../src/rag_pipeline/core/source_tree.py | 382 +++++++++++ .../src/rag_pipeline/models/config.py | 3 + .../rag_pipeline/server/rag_queue_consumer.py | 28 +- .../src/rag_pipeline/services/base.py | 2 +- .../services/deterministic_context.py | 147 ++++- .../src/rag_pipeline/services/pr_context.py | 18 +- .../rag_pipeline/services/semantic_search.py | 48 +- .../rag-pipeline/tests/test_coordination.py | 27 + .../tests/test_generation_advance.py | 258 ++++++++ .../rag-pipeline/tests/test_index_manager.py | 44 +- .../rag-pipeline/tests/test_indexer.py | 31 +- .../tests/test_loader_extended.py | 5 +- .../tests/test_pr_overlay_manifest.py | 274 ++++++++ .../tests/test_revision_bound_queries.py | 528 +++++++++++++++ .../tests/test_revision_preflight.py | 604 ++++++++++++++++++ .../rag-pipeline/tests/test_router_index.py | 103 +++ .../rag-pipeline/tests/test_source_tree.py | 156 +++++ 148 files changed, 12755 insertions(+), 430 deletions(-) create mode 100644 java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGeneration.java create mode 100644 java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationStatus.java create mode 100644 java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java create mode 100644 java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexLifecycleStatus.java create mode 100644 java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java create mode 100644 java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperationStatus.java create mode 100644 java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java create mode 100644 java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java create mode 100644 java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.23.0__exact_branch_index_registry.sql create mode 100644 java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.24.0__scm_evidence_and_branch_analysis_scope.sql create mode 100644 java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java create mode 100644 java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java create mode 100644 java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java create mode 100644 java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java create mode 100644 java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java create mode 100644 java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java create mode 100644 java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java create mode 100644 java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java create mode 100644 java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentity.java create mode 100644 java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java create mode 100644 java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java create mode 100644 java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java create mode 100644 java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java create mode 100644 java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java create mode 100644 java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentityTest.java create mode 100644 java-ecosystem/libs/scm-evidence/pom.xml create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/module-info.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/AnalysisReceiptView.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/CommitEvidenceView.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/IssueProvenance.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/PromotionPlan.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfiguration.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAddedLineEvidence.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAnalysisReceipt.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmCommitEvidence.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAddedLineEvidenceRepository.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAnalysisReceiptRepository.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmCommitEvidenceRepository.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/PatchIdentity.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceService.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlanner.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParser.java create mode 100644 java-ecosystem/libs/scm-evidence/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports create mode 100644 java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfigurationTest.java create mode 100644 java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceServiceTest.java create mode 100644 java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlannerTest.java create mode 100644 java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParserTest.java create mode 100644 java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/response/RagBranchIndexStatusDTO.java create mode 100644 java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusService.java create mode 100644 java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusServiceTest.java create mode 100644 python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py create mode 100644 python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_manifest.py create mode 100644 python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_binding.py create mode 100644 python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py create mode 100644 python-ecosystem/rag-pipeline/src/rag_pipeline/core/source_tree.py create mode 100644 python-ecosystem/rag-pipeline/tests/test_generation_advance.py create mode 100644 python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py create mode 100644 python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py create mode 100644 python-ecosystem/rag-pipeline/tests/test_revision_preflight.py create mode 100644 python-ecosystem/rag-pipeline/tests/test_source_tree.py diff --git a/deployment/config/java-shared/application.properties.sample b/deployment/config/java-shared/application.properties.sample index 2cc383e71..bbc17065c 100644 --- a/deployment/config/java-shared/application.properties.sample +++ b/deployment/config/java-shared/application.properties.sample @@ -181,6 +181,11 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF #codecrow.rag.api.timeout.read=120 # RAG indexing timeout - 4 hours for large repositories #codecrow.rag.api.timeout.indexing=14400 +# Number of configured branch snapshots that an explicit Refresh all may build in parallel. +#codecrow.rag.branch-build.parallelism=4 +# Repair interval for readable Qdrant current-branch aliases of active generations. +#codecrow.rag.operator-alias.reconcile-interval-ms=300000 +#codecrow.rag.operator-alias.reconcile-initial-delay-ms=15000 # Shared VCS acquisition threshold for incremental RAG and reconciliation # fallback. The legacy codecrow.rag.incremental.archive-file-threshold property # remains a fallback when this property is not set. diff --git a/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java b/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java index 86d4341a1..3b8823253 100644 --- a/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java +++ b/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java @@ -118,6 +118,20 @@ default boolean shouldHaveBranchIndex(Project project, String branchName) { : null; return config.ragConfig().shouldHaveBranchIndex(branchName, branchPushPatterns); } + + /** + * Whether an eligible PR target that is not retained may receive a temporary, + * revision-pinned branch snapshot. This never makes branch pushes retain data. + */ + default boolean shouldCreateTransientBranchIndex(Project project, String branchName) { + var config = project.getConfiguration(); + if (config == null || config.ragConfig() == null || branchName == null) { + return false; + } + return config.ragConfig().isTransientBranchIndexesEnabled() + && !branchName.equals(getBaseBranch(project)) + && !shouldHaveBranchIndex(project, branchName); + } /** * Get the authoritative base branch for RAG indexing. @@ -203,11 +217,9 @@ default void createOrUpdateBranchIndex( } /** - * Update branch index by calculating diff between base branch and target branch. - * - * This method always recalculates the full diff between the base branch (e.g., "master") - * and the target branch (e.g., "release/1.0"), then indexes all changed files with - * the target branch in their metadata. + * Update an already retained branch from its completed checkpoint. A first + * legacy branch seed may still compare it with the primary branch; exact + * generation implementations replace that seed path with a complete snapshot. * * Use this when a push happens to a non-main branch and you need to update * the RAG index to reflect the current state of that branch. diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java index 7de7ef5ca..58642f6d8 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java @@ -10,6 +10,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.rostilos.codecrow.queue.RedisQueueService; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; @@ -35,6 +37,9 @@ public class AiAnalysisClient { private final RedisQueueService queueService; private final ObjectMapper objectMapper; + @Autowired(required = false) + private RagBranchIndexGenerationRepository branchGenerationRepository; + static final String INACTIVITY_TIMEOUT_MINUTES_KEY = "ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES"; static final String ADMISSION_TIMEOUT_MINUTES_KEY = @@ -348,6 +353,25 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("previousCommitHash", request.getPreviousCommitHash()); payload.put("currentCommitHash", request.getCurrentCommitHash()); payload.put("baseCommitHash", request.getBaseCommitHash()); + if (branchGenerationRepository != null + && request.getProjectId() != null + && request.getTargetBranchName() != null + && request.getBaseCommitHash() != null) { + branchGenerationRepository.findAvailableExactGeneration( + request.getProjectId(), + request.getTargetBranchName(), + request.getBaseCommitHash(), + List.of( + RagBranchIndexGenerationStatus.ACTIVE, + RagBranchIndexGenerationStatus.SUPERSEDED)) + .stream() + .findFirst() + .ifPresent(generation -> { + payload.put("ragCollectionTarget", generation.getCollectionName()); + payload.put("ragBaseGenerationManifestSha256", + generation.getManifestDigest()); + }); + } payload.put("previousCodeAnalysisIssues", request.getPreviousCodeAnalysisIssues()); payload.put("reconciliationFileContents", request.getReconciliationFileContents()); payload.put("projectCapabilities", request.getProjectCapabilities()); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java index 1084fa83c..21d34487b 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java @@ -724,8 +724,12 @@ private void performDirectPushAnalysisIfNeeded( } // Check commit coverage by open/merged PRs + boolean exactTargetBranchCoverage = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); CommitCoverageService.CoverageResult coverage = commitCoverageService.checkCoverage( - project.getId(), request.getTargetBranchName(), unanalyzedCommits); + project.getId(), request.getTargetBranchName(), unanalyzedCommits, + exactTargetBranchCoverage); switch (coverage.status()) { case FULLY_COVERED: @@ -858,10 +862,19 @@ private void performIncrementalRagUpdate(BranchProcessRequest request, Project p return; } - String targetBranch = request.getTargetBranchName(); - String baseBranch = ragOperationsService.getBaseBranch(project); + String targetBranch = request.getTargetBranchName(); + String baseBranch = ragOperationsService.getBaseBranch(project); - // Health check: verify RAG pipeline is reachable before starting + if (!targetBranch.equals(baseBranch) + && !ragOperationsService.shouldHaveBranchIndex(project, targetBranch)) { + log.info("Skipping RAG update for non-retained branch: project={}, branch={}", + project.getId(), targetBranch); + EventNotificationEmitter.emitStatus(consumer, "rag_skipped", + "Branch is analyzed but is not configured as a retained RAG branch"); + return; + } + + // Health check: verify RAG pipeline is reachable before starting if (!ragOperationsService.isRagPipelineHealthy()) { log.warn("RAG pipeline is not reachable — skipping incremental update for project={}", project.getId()); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java index 023c7a0f9..e6fea6caf 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java @@ -4,6 +4,7 @@ import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue; +import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.pullrequest.PullRequest; import org.rostilos.codecrow.core.model.vcs.EVcsProvider; @@ -52,6 +53,9 @@ import org.rostilos.codecrow.analysisengine.util.PromptDryRunMode; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.rostilos.codecrow.vcsclient.model.VcsCommit; +import org.rostilos.codecrow.scmevidence.service.ScmEvidenceService; +import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisIssueRepository; /** * Generic service that handles pull request analysis. @@ -61,6 +65,7 @@ @Service public class PullRequestAnalysisProcessor { private static final Logger log = LoggerFactory.getLogger(PullRequestAnalysisProcessor.class); + private static final int SCM_EVIDENCE_COMMIT_LIMIT = 1000; private final CodeAnalysisService codeAnalysisService; private final TaskImplementationEvidenceService taskImplementationEvidenceService; @@ -76,6 +81,12 @@ public class PullRequestAnalysisProcessor { private final PrIssueTrackingService prIssueTrackingService; private final AstScopeEnricher astScopeEnricher; + @Autowired(required = false) + private ScmEvidenceService scmEvidenceService; + + @Autowired(required = false) + private CodeAnalysisIssueRepository codeAnalysisIssueRepository; + public PullRequestAnalysisProcessor( PullRequestService pullRequestService, CodeAnalysisService codeAnalysisService, @@ -357,7 +368,7 @@ public Map process( } // === DAG: Mark PR commits as ANALYZED === - markPrCommitsAnalyzed(project, request.getSourceBranchName(), request.getCommitHash(), newAnalysis); + markPrCommitsAnalyzed(project, request, newAnalysis); // Publish successful completion event publishAnalysisCompletedEvent(project, request, correlationId, startTime, @@ -741,13 +752,95 @@ private static void putIdentity(Map target, String key, Object v * @param commitHash the HEAD commit of the source branch * @param analysis the CodeAnalysis to link, or null for cache-hit scenarios */ - private void markPrCommitsAnalyzed(Project project, String sourceBranch, String commitHash, CodeAnalysis analysis) { + private void markPrCommitsAnalyzed( + Project project, + PrProcessRequest request, + CodeAnalysis analysis) { try { + String sourceBranch = request.getSourceBranchName(); + String targetBranch = request.getTargetBranchName(); + String commitHash = request.getCommitHash(); if (commitHash == null) return; + String targetBaseRevision = null; + List analyzedHashes = List.of(commitHash); + if (scmEvidenceService != null) { + try { + VcsClient client = vcsClientProvider.getClient( + project.getEffectiveVcsRepoInfo().getVcsConnection()); + String workspace = project.getEffectiveVcsRepoInfo().getRepoWorkspace(); + String repository = project.getEffectiveVcsRepoInfo().getRepoSlug(); + var pullRequest = client.getPullRequest( + workspace, repository, request.getPullRequestId()); + targetBaseRevision = pullRequest != null + ? pullRequest.baseCommit() : null; + List newestFirst = client.getCommitHistory( + workspace, repository, sourceBranch, + SCM_EVIDENCE_COMMIT_LIMIT); + List prCommits = new java.util.ArrayList<>(); + for (VcsCommit commit : newestFirst == null + ? List.of() : newestFirst) { + if (targetBaseRevision != null + && targetBaseRevision.equals(commit.hash())) { + break; + } + prCommits.add(commit); + if (commitHash.equals(commit.hash()) + && targetBaseRevision == null) { + break; + } + } + Collections.reverse(prCommits); + if (!prCommits.isEmpty()) { + scmEvidenceService.capture( + project.getId(), client, workspace, repository, + prCommits); + analyzedHashes = prCommits.stream() + .map(VcsCommit::hash) + .toList(); + } + scmEvidenceService.recordAnalysisReceipts( + project.getId(), analyzedHashes, + sourceBranch, targetBranch, targetBaseRevision, + analysis != null ? analysis.getId() : null, + AnalysisType.PR_REVIEW.name()); + if (analysis != null && codeAnalysisIssueRepository != null) { + for (CodeAnalysisIssue issue : analysis.getIssues()) { + scmEvidenceService.resolveIssueProvenance( + project.getId(), analyzedHashes, + issue.getFilePath(), issue.getLineNumber(), + issue.getCodeSnippet()) + .ifPresent(provenance -> { + issue.setIntroducingCommitHash( + provenance.commitHash()); + issue.setIntroducingAuthorName( + provenance.authorName()); + issue.setIntroducingAuthorEmail( + provenance.authorEmail()); + issue.setAuthorProvenanceConfidence( + provenance.confidence()); + }); + } + codeAnalysisIssueRepository.saveAll(analysis.getIssues()); + } + } catch (Exception evidenceFailure) { + log.warn("SCM promotion/provenance evidence unavailable for PR #{}: {}", + request.getPullRequestId(), evidenceFailure.getMessage()); + } + } + // Record the PR's HEAD commit as analyzed - analyzedCommitService.recordPrCommitsAnalyzed( - project, List.of(commitHash), analysis); + boolean multiBranch = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + if (multiBranch) { + analyzedCommitService.recordPrCommitsAnalyzed( + project, analyzedHashes, analysis, + sourceBranch, targetBranch, targetBaseRevision); + } else { + analyzedCommitService.recordPrCommitsAnalyzed( + project, List.of(commitHash), analysis); + } log.info("Recorded PR commit {} as analyzed (branch={}, analysis={})", commitHash.substring(0, Math.min(7, commitHash.length())), @@ -755,7 +848,7 @@ private void markPrCommitsAnalyzed(Project project, String sourceBranch, String analysis != null ? analysis.getId() : "none"); } catch (Exception e) { log.warn("Failed to record PR commit as analyzed (non-critical): branch={}, error={}", - sourceBranch, e.getMessage()); + request.getSourceBranchName(), e.getMessage()); } } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java index 9863323a9..802369aa6 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java @@ -30,8 +30,9 @@ public class BranchArchiveService { private static final Logger log = LoggerFactory.getLogger(BranchArchiveService.class); /** - * Maximum single-file size to extract from archive (10 MB). - * Files larger than this are skipped to avoid memory pressure. + * Maximum single-file size retained by the in-memory extraction API. + * Directory extraction, used by RAG indexing, streams entries directly to + * disk and does not impose this limit on an archive or its members. */ private static final long MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; @@ -164,12 +165,11 @@ public ArchiveDirectorySnapshot downloadAndExtractSnapshotToDirectory( repoSlug, branchOrCommit, archiveFile -> { - Set extractedFiles = extractFilesFromArchive( + Set extractedFiles = extractFilesToDirectory( archiveFile, neededFiles, presentFiles, - (relativePath, bytes) -> - writeFile(normalizedTarget, relativePath, bytes)); + normalizedTarget); return new ArchiveDirectorySnapshot(extractedFiles, presentFiles); }); } @@ -244,16 +244,19 @@ private Set extractFilesFromArchive( presentFiles.add(relativePath); - // Read the entry content - byte[] bytes = readZipEntry(zis); + // The in-memory API must not retain an unbounded archive + // member. RAG uses extractFilesToDirectory below instead, + // which streams every accepted entry to disk. + ZipEntryContent content = readZipEntryBounded(zis); - // Skip very large files - if (bytes.length > MAX_FILE_SIZE_BYTES) { - log.debug("Skipping large file {} ({} bytes)", relativePath, bytes.length); + if (content.tooLarge()) { + log.debug("Skipping large file {} (more than {} bytes)", + relativePath, MAX_FILE_SIZE_BYTES); skippedLarge++; zis.closeEntry(); continue; } + byte[] bytes = content.bytes(); // Skip binary files (null bytes in first 1 KB) if (isBinary(bytes)) { @@ -285,17 +288,91 @@ private Set extractFilesFromArchive( return extractedFiles; } - private boolean writeFile(Path targetDirectory, String relativePath, byte[] bytes) throws IOException { + /** + * Streams archive entries directly into an isolated repository directory. + * This is the RAG path: it intentionally has no archive-size or entry-size + * ceiling, so large repositories do not require correspondingly large JVM + * heaps. Only a small prefix and a fixed copy buffer are held in memory. + */ + private Set extractFilesToDirectory( + Path archiveFile, + Set neededFiles, + Set presentFiles, + Path targetDirectory + ) throws IOException { + Set extractedFiles = new LinkedHashSet<>(); + int skippedBinary = 0; + int skippedNotNeeded = 0; + int skippedUnsafe = 0; + + try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(archiveFile))) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.isDirectory()) { + zis.closeEntry(); + continue; + } + + String relativePath = stripArchiveRoot(entry.getName()); + if (neededFiles != null && !neededFiles.isEmpty() + && !neededFiles.contains(relativePath)) { + skippedNotNeeded++; + zis.closeEntry(); + continue; + } + + presentFiles.add(relativePath); + Path targetFile = resolveTargetFile(targetDirectory, relativePath); + if (targetFile == null) { + skippedUnsafe++; + zis.closeEntry(); + continue; + } + + byte[] prefix = zis.readNBytes(1024); + if (isBinary(prefix, prefix.length)) { + skippedBinary++; + zis.closeEntry(); + continue; + } + + try (OutputStream output = Files.newOutputStream(targetFile)) { + output.write(prefix); + byte[] buffer = new byte[8192]; + int length; + while ((length = zis.read(buffer)) > 0) { + output.write(buffer, 0, length); + } + } + setWorldReadable(targetFile, false); + extractedFiles.add(relativePath); + zis.closeEntry(); + + if (neededFiles != null && !neededFiles.isEmpty() + && extractedFiles.size() >= neededFiles.size()) { + break; + } + } + } + + log.info("Archive extraction: {} files streamed to disk, {} skipped (not needed), " + + "{} binary, {} unsafe. Requested: {}", + extractedFiles.size(), skippedNotNeeded, skippedBinary, skippedUnsafe, + neededFiles != null ? neededFiles.size() : "all"); + return extractedFiles; + } + + private Path resolveTargetFile(Path targetDirectory, String relativePath) throws IOException { Path targetFile; try { targetFile = targetDirectory.resolve(relativePath).normalize(); } catch (RuntimeException e) { log.warn("Skipping invalid archive entry path: {}", relativePath); - return false; + return null; } if (!targetFile.startsWith(targetDirectory) || targetFile.equals(targetDirectory)) { log.warn("Skipping archive entry outside target directory: {}", relativePath); - return false; + return null; } Path parent = targetFile.getParent(); @@ -305,9 +382,7 @@ private boolean writeFile(Path targetDirectory, String relativePath, byte[] byte directory = directory.getParent()) { setWorldReadable(directory, true); } - Files.write(targetFile, bytes); - setWorldReadable(targetFile, false); - return true; + return targetFile; } private void setWorldReadable(Path path, boolean directory) { @@ -338,14 +413,23 @@ static String stripArchiveRoot(String entryPath) { return entryPath; } - private byte[] readZipEntry(ZipInputStream zis) throws IOException { + private ZipEntryContent readZipEntryBounded(ZipInputStream zis) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); byte[] buffer = new byte[8192]; + long totalBytes = 0; + boolean tooLarge = false; int len; while ((len = zis.read(buffer)) > 0) { + totalBytes += len; + if (totalBytes > MAX_FILE_SIZE_BYTES) { + tooLarge = true; + continue; + } baos.write(buffer, 0, len); } - return baos.toByteArray(); + return tooLarge + ? ZipEntryContent.TOO_LARGE + : new ZipEntryContent(baos.toByteArray(), false); } /** @@ -353,7 +437,11 @@ private byte[] readZipEntry(ZipInputStream zis) throws IOException { * in the first 1024 bytes. */ private boolean isBinary(byte[] data) { - int checkLimit = Math.min(data.length, 1024); + return isBinary(data, data.length); + } + + private boolean isBinary(byte[] data, int length) { + int checkLimit = Math.min(length, 1024); for (int i = 0; i < checkLimit; i++) { if (data[i] == 0) return true; } @@ -376,6 +464,10 @@ private interface ExtractedFileConsumer { boolean accept(String relativePath, byte[] bytes) throws IOException; } + private record ZipEntryContent(byte[] bytes, boolean tooLarge) { + private static final ZipEntryContent TOO_LARGE = new ZipEntryContent(null, true); + } + public record ArchiveSnapshot( Map contents, Set presentFiles diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java index 5591e31f4..5831375ff 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java @@ -2,10 +2,13 @@ import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; import org.rostilos.codecrow.commitgraph.service.AnalyzedCommitService; +import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.persistence.repository.branch.BranchRepository; +import org.rostilos.codecrow.scmevidence.service.ScmEvidenceService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.List; @@ -17,6 +20,9 @@ public class BranchHealthService { private final BranchRepository branchRepository; private final AnalyzedCommitService analyzedCommitService; + @Autowired(required = false) + private ScmEvidenceService scmEvidenceService; + public BranchHealthService( BranchRepository branchRepository, AnalyzedCommitService analyzedCommitService @@ -37,7 +43,22 @@ public void recordCommitsAnalyzed(Project project, List unanalyzedCommit String branchName) { if (unanalyzedCommits.isEmpty()) return; try { - analyzedCommitService.recordBranchCommitsAnalyzed(project, unanalyzedCommits); + boolean multiBranch = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + if (multiBranch) { + analyzedCommitService.recordBranchCommitsAnalyzed( + project, unanalyzedCommits, branchName); + } else { + analyzedCommitService.recordBranchCommitsAnalyzed( + project, unanalyzedCommits); + } + if (scmEvidenceService != null) { + scmEvidenceService.recordAnalysisReceipts( + project.getId(), unanalyzedCommits, branchName, branchName, + unanalyzedCommits.get(unanalyzedCommits.size() - 1), + null, AnalysisType.BRANCH_ANALYSIS.name()); + } log.info("Recorded {} commits as analyzed after successful branch analysis (branch={})", unanalyzedCommits.size(), branchName); } catch (Exception e) { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java index ec305311f..69ed3606d 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java @@ -318,6 +318,47 @@ void shouldIncludeSourceAndTargetBranchNamesInQueuedRequestPayload() throws Exce assertThat(requestPayload.get("ragEnabled")).isEqualTo(true); } + @Test + @DisplayName("should bind exact target branch generation to queued review") + void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { + var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexGenerationRepository.class); + var generation = mock(org.rostilos.codecrow.core.model.rag + .RagBranchIndexGeneration.class); + when(generation.getCollectionName()).thenReturn("opaque-master-generation"); + when(generation.getManifestDigest()).thenReturn("master-manifest"); + when(repository.findAvailableExactGeneration( + eq(1L), eq("main"), eq("master-base"), anyList())) + .thenReturn(List.of(generation)); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchGenerationRepository", repository); + AiAnalysisRequest exactRequest = new TestAiAnalysisRequest() { + @Override + public String getBaseCommitHash() { + return "master-base"; + } + }; + Map finalEvent = Map.of( + "type", "final", + "result", Map.of("comment", "ok", "issues", List.of())); + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(finalEvent)); + + client.performAnalysis(exactRequest); + + var payloadCaptor = org.mockito.ArgumentCaptor.forClass(String.class); + verify(queueService).leftPush(eq("codecrow:analysis:jobs"), payloadCaptor.capture()); + @SuppressWarnings("unchecked") + Map queued = objectMapper.readValue( + payloadCaptor.getValue(), Map.class); + @SuppressWarnings("unchecked") + Map requestPayload = + (Map) queued.get("request"); + assertThat(requestPayload) + .containsEntry("ragCollectionTarget", "opaque-master-generation") + .containsEntry("ragBaseGenerationManifestSha256", "master-manifest"); + } + @Test @DisplayName("should include task context in queued request payload") void shouldIncludeTaskContextInQueuedRequestPayload() throws Exception { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java index f987a06ee..5781c71f8 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java @@ -586,6 +586,7 @@ void shouldScopeOversizedDirectPushToFilesWithPreviousIssues() throws Exception when(ragOperationsService.isRagEnabled(project)).thenReturn(true); when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); + when(ragOperationsService.shouldHaveBranchIndex(project, "feature-x")).thenReturn(true); when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true); Map result = processor.process(request, events::add); @@ -807,6 +808,7 @@ void shouldCallUpdateBranchIndexForNonMainBranch() throws Exception { when(ragOperationsService.isRagEnabled(project)).thenReturn(true); when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); + when(ragOperationsService.shouldHaveBranchIndex(project, "feature-x")).thenReturn(true); when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true); when(branchRepository.findByProjectIdAndBranchName(1L, "feature-x")) @@ -819,6 +821,36 @@ void shouldCallUpdateBranchIndexForNonMainBranch() throws Exception { verify(ragOperationsService, never()).triggerIncrementalUpdate(any(), any(), any(), any(), any()); } + @Test + @DisplayName("should not mutate RAG for an analyzed branch that is not retained") + void shouldSkipRagForNonRetainedBranch() { + BranchProcessRequest request = createRequest(); + request.targetBranchName = "release/preview"; + request.commitHash = "release-commit"; + List> events = new ArrayList<>(); + + when(project.getId()).thenReturn(1L); + when(ragOperationsService.isRagEnabled(project)).thenReturn(true); + when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); + when(ragOperationsService.getBaseBranch(project)).thenReturn("master"); + when(ragOperationsService.shouldHaveBranchIndex(project, "release/preview")).thenReturn(false); + + ReflectionTestUtils.invokeMethod( + processor, + "performIncrementalRagUpdate", + request, + project, + "diff --git a/f.java b/f.java\n+x\n", + (Consumer>) events::add, + false); + + verify(ragOperationsService, never()).isRagPipelineHealthy(); + verify(ragOperationsService, never()).updateBranchIndex(any(), any(), any()); + verify(ragOperationsService, never()).triggerIncrementalUpdate(any(), any(), any(), any(), any()); + assertThat(events).anySatisfy(event -> + assertThat(event).containsEntry("state", "rag_skipped")); + } + @Test @DisplayName("should handle delta diff failure and fall back to PR diff") void shouldFallBackToPrDiffWhenDeltaDiffFails() throws Exception { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java index d0e8e16d4..00c1a03be 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java @@ -185,6 +185,38 @@ void shouldSkipBinaryFiles(@TempDir Path tempDir) throws Exception { assertThat(result).doesNotContainKey("binary.dat"); } + @Test + void shouldStreamLargeArchiveEntriesToDiskWithoutRetainingThem(@TempDir Path tempDir) + throws Exception { + VcsConnection conn = new VcsConnection(); + when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); + + byte[] largeFile = new byte[10 * 1024 * 1024 + 1]; + java.util.Arrays.fill(largeFile, (byte) 'x'); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + zos.putNextEntry(new ZipEntry("root/large.generated")); + zos.write(largeFile); + zos.closeEntry(); + zos.putNextEntry(new ZipEntry("root/src/StillIndexed.java")); + zos.write("class StillIndexed {}".getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + } + byte[] zipBytes = baos.toByteArray(); + when(vcsClient.downloadRepositoryArchiveToFile(anyString(), anyString(), anyString(), any(Path.class))) + .thenAnswer(inv -> { + Files.write(inv.getArgument(3, Path.class), zipBytes); + return (long) zipBytes.length; + }); + + Set extracted = service.downloadAndExtractFilesToDirectory( + conn, "ws", "repo", "main", null, tempDir.resolve("repository")); + + assertThat(extracted).containsExactlyInAnyOrder("large.generated", "src/StillIndexed.java"); + assertThat(tempDir.resolve("repository/large.generated")).hasSize(largeFile.length); + assertThat(tempDir.resolve("repository/src/StillIndexed.java")).exists(); + } + @Test void snapshotReportsBinaryPathPresenceWithoutLoadingItsContent(@TempDir Path tempDir) throws Exception { diff --git a/java-ecosystem/libs/commit-graph/pom.xml b/java-ecosystem/libs/commit-graph/pom.xml index a598ff59a..40fda6bd6 100644 --- a/java-ecosystem/libs/commit-graph/pom.xml +++ b/java-ecosystem/libs/commit-graph/pom.xml @@ -33,6 +33,11 @@ codecrow-vcs-client + + org.rostilos.codecrow + codecrow-scm-evidence + + org.springframework.boot diff --git a/java-ecosystem/libs/commit-graph/src/main/java/module-info.java b/java-ecosystem/libs/commit-graph/src/main/java/module-info.java index 903c7bbe0..fd08bc424 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/module-info.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/module-info.java @@ -10,6 +10,7 @@ requires org.rostilos.codecrow.core; requires org.rostilos.codecrow.vcs; + requires org.rostilos.codecrow.scmevidence; // Model exports exports org.rostilos.codecrow.commitgraph.model; diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/model/AnalyzedCommit.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/model/AnalyzedCommit.java index 2cb286296..16c711f4f 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/model/AnalyzedCommit.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/model/AnalyzedCommit.java @@ -19,12 +19,6 @@ @Entity @Table( name = "analyzed_commit", - uniqueConstraints = { - @UniqueConstraint( - name = "uq_analyzed_commit_project_hash", - columnNames = {"project_id", "commit_hash"} - ) - }, indexes = { @Index(name = "idx_analyzed_commit_project", columnList = "project_id"), @Index(name = "idx_analyzed_commit_hash", columnList = "commit_hash") @@ -61,6 +55,15 @@ public class AnalyzedCommit { @Column(name = "analysis_type", length = 30) private AnalysisType analysisType; + @Column(name = "source_branch", length = 256) + private String sourceBranch; + + @Column(name = "target_branch", length = 256) + private String targetBranch; + + @Column(name = "target_base_revision", length = 64) + private String targetBaseRevision; + // ── Constructors ─────────────────────────────────────────────────── public AnalyzedCommit() { @@ -97,4 +100,10 @@ public AnalyzedCommit(Project project, String commitHash, Long analysisId, Analy public AnalysisType getAnalysisType() { return analysisType; } public void setAnalysisType(AnalysisType analysisType) { this.analysisType = analysisType; } + public String getSourceBranch() { return sourceBranch; } + public void setSourceBranch(String sourceBranch) { this.sourceBranch = sourceBranch; } + public String getTargetBranch() { return targetBranch; } + public void setTargetBranch(String targetBranch) { this.targetBranch = targetBranch; } + public String getTargetBaseRevision() { return targetBaseRevision; } + public void setTargetBaseRevision(String targetBaseRevision) { this.targetBaseRevision = targetBaseRevision; } } diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/persistence/AnalyzedCommitRepository.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/persistence/AnalyzedCommitRepository.java index 47460eab3..b4f5243a2 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/persistence/AnalyzedCommitRepository.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/persistence/AnalyzedCommitRepository.java @@ -18,6 +18,9 @@ public interface AnalyzedCommitRepository extends JpaRepository findAnalyzedHashesByProjectIdAndCommitHashIn( @Param("projectId") Long projectId, @Param("hashes") List hashes); + @Query(""" + SELECT ac.commitHash FROM AnalyzedCommit ac + WHERE ac.project.id = :projectId + AND ac.targetBranch = :targetBranch + AND ac.commitHash IN :hashes + """) + Set findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + @Param("projectId") Long projectId, + @Param("targetBranch") String targetBranch, + @Param("hashes") List hashes); + /** * Find all analyzed commits for a project. */ diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/AnalyzedCommitService.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/AnalyzedCommitService.java index 31a9c93d4..3ff715b73 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/AnalyzedCommitService.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/AnalyzedCommitService.java @@ -41,15 +41,29 @@ public AnalyzedCommitService(AnalyzedCommitRepository analyzedCommitRepository) */ @Transactional public void recordBranchCommitsAnalyzed(Project project, List hashes) { + recordBranchCommitsAnalyzed(project, hashes, null); + } + + @Transactional + public void recordBranchCommitsAnalyzed( + Project project, List hashes, String targetBranch) { if (hashes == null || hashes.isEmpty()) return; - Set alreadyAnalyzed = analyzedCommitRepository - .findAnalyzedHashesByProjectIdAndCommitHashIn(project.getId(), hashes); + Set alreadyAnalyzed = targetBranch == null + ? analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndCommitHashIn( + project.getId(), hashes) + : analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + project.getId(), targetBranch, hashes); List toSave = new ArrayList<>(); for (String hash : hashes) { if (!alreadyAnalyzed.contains(hash)) { - toSave.add(new AnalyzedCommit(project, hash, AnalysisType.BRANCH_ANALYSIS)); + AnalyzedCommit analyzed = new AnalyzedCommit( + project, hash, AnalysisType.BRANCH_ANALYSIS); + analyzed.setTargetBranch(targetBranch); + toSave.add(analyzed); } } @@ -69,16 +83,37 @@ public void recordBranchCommitsAnalyzed(Project project, List hashes) { */ @Transactional public void recordPrCommitsAnalyzed(Project project, List hashes, CodeAnalysis analysis) { + recordPrCommitsAnalyzed(project, hashes, analysis, null, null, null); + } + + @Transactional + public void recordPrCommitsAnalyzed( + Project project, + List hashes, + CodeAnalysis analysis, + String sourceBranch, + String targetBranch, + String targetBaseRevision) { if (hashes == null || hashes.isEmpty()) return; - Set alreadyAnalyzed = analyzedCommitRepository - .findAnalyzedHashesByProjectIdAndCommitHashIn(project.getId(), hashes); + Set alreadyAnalyzed = targetBranch == null + ? analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndCommitHashIn( + project.getId(), hashes) + : analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + project.getId(), targetBranch, hashes); Long analysisId = analysis != null ? analysis.getId() : null; List toSave = new ArrayList<>(); for (String hash : hashes) { if (!alreadyAnalyzed.contains(hash)) { - toSave.add(new AnalyzedCommit(project, hash, analysisId, AnalysisType.PR_REVIEW)); + AnalyzedCommit analyzed = new AnalyzedCommit( + project, hash, analysisId, AnalysisType.PR_REVIEW); + analyzed.setSourceBranch(sourceBranch); + analyzed.setTargetBranch(targetBranch); + analyzed.setTargetBaseRevision(targetBaseRevision); + toSave.add(analyzed); } } @@ -107,10 +142,26 @@ public List filterUnanalyzed(Long projectId, List allHashes) { .toList(); } + public List filterUnanalyzed( + Long projectId, String targetBranch, List allHashes) { + if (allHashes == null || allHashes.isEmpty()) return List.of(); + Set analyzed = analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + projectId, targetBranch, allHashes); + return allHashes.stream().filter(hash -> !analyzed.contains(hash)).toList(); + } + /** * Check if a specific commit has been analyzed. */ public boolean isAnalyzed(Long projectId, String commitHash) { return analyzedCommitRepository.existsByProjectIdAndCommitHash(projectId, commitHash); } + + public boolean isAnalyzed( + Long projectId, String targetBranch, String commitHash) { + return analyzedCommitRepository + .existsByProjectIdAndTargetBranchAndCommitHash( + projectId, targetBranch, commitHash); + } } diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java index 64668b777..ecc423a38 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java @@ -8,12 +8,15 @@ import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.rostilos.codecrow.vcsclient.model.VcsCommit; +import org.rostilos.codecrow.scmevidence.service.ScmEvidenceService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -34,20 +37,34 @@ public class BranchCommitService { private static final Logger log = LoggerFactory.getLogger(BranchCommitService.class); - private static final int DEFAULT_COMMIT_FETCH_LIMIT = 100; + // Preserve evidence for long-lived branches instead of collapsing a 400+ + // commit promotion to a HEAD-only fallback. + private static final int DEFAULT_COMMIT_FETCH_LIMIT = 1000; private final VcsClientProvider vcsClientProvider; private final AnalyzedCommitService analyzedCommitService; private final BranchRepository branchRepository; + private final ScmEvidenceService scmEvidenceService; public BranchCommitService( VcsClientProvider vcsClientProvider, AnalyzedCommitService analyzedCommitService, BranchRepository branchRepository + ) { + this(vcsClientProvider, analyzedCommitService, branchRepository, null); + } + + @Autowired + public BranchCommitService( + VcsClientProvider vcsClientProvider, + AnalyzedCommitService analyzedCommitService, + BranchRepository branchRepository, + ScmEvidenceService scmEvidenceService ) { this.vcsClientProvider = vcsClientProvider; this.analyzedCommitService = analyzedCommitService; this.branchRepository = branchRepository; + this.scmEvidenceService = scmEvidenceService; } /** @@ -82,7 +99,15 @@ public CommitRangeContext resolveCommitRange( // ── Same commit: nothing to do ─────────────────────────────────── if (lastKnownHead.equals(commitHash)) { // Check if this commit is already analyzed - if (analyzedCommitService.isAnalyzed(project.getId(), commitHash)) { + boolean multiBranch = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + boolean analyzed = multiBranch + ? analyzedCommitService.isAnalyzed( + project.getId(), targetBranchName, commitHash) + : analyzedCommitService.isAnalyzed( + project.getId(), commitHash); + if (analyzed) { log.info("HEAD commit {} is already analyzed — skipping", shortHash(commitHash)); return CommitRangeContext.skip(); } @@ -130,9 +155,41 @@ public CommitRangeContext resolveCommitRange( // Reverse to chronological order (oldest first) Collections.reverse(commitsSinceLastHead); + if (scmEvidenceService != null) { + try { + Map commitsByHash = commits.stream() + .collect(java.util.stream.Collectors.toMap( + VcsCommit::hash, + commit -> commit, + (first, ignored) -> first)); + List evidenceCommits = commitsSinceLastHead.stream() + .map(commitsByHash::get) + .filter(java.util.Objects::nonNull) + .toList(); + scmEvidenceService.capture( + project.getId(), vcsClient, workspace, slug, + evidenceCommits); + var promotion = scmEvidenceService.planPromotion( + project.getId(), commitsSinceLastHead, + targetBranchName, lastKnownHead); + log.info("SCM promotion evidence for branch {}: reuse={}, targetContextAnalysis={}", + targetBranchName, promotion.reuseKind(), + promotion.requiresTargetContextAnalysis()); + } catch (Exception evidenceFailure) { + log.warn("SCM evidence enrichment unavailable for branch {}: {}", + targetBranchName, evidenceFailure.getMessage()); + } + } + // Subtract already-analyzed commits - List unanalyzed = analyzedCommitService.filterUnanalyzed( - project.getId(), commitsSinceLastHead); + boolean multiBranch = project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + List unanalyzed = multiBranch + ? analyzedCommitService.filterUnanalyzed( + project.getId(), targetBranchName, commitsSinceLastHead) + : analyzedCommitService.filterUnanalyzed( + project.getId(), commitsSinceLastHead); if (unanalyzed.isEmpty()) { log.info("All {} commits since lastKnownHead are already analyzed — skipping", diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageService.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageService.java index a49d7daf4..50d68efa8 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageService.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageService.java @@ -78,13 +78,30 @@ public enum CoverageStatus { */ public CoverageResult checkCoverage(Long projectId, String targetBranchName, List unanalyzedCommits) { + return checkCoverage(projectId, targetBranchName, unanalyzedCommits, false); + } + + /** + * Check coverage while optionally limiting durable receipts to the exact + * target branch. This prevents an analysis on {@code develop} from + * suppressing analysis of the same commits in a different {@code master} + * context. + */ + public CoverageResult checkCoverage(Long projectId, String targetBranchName, + List unanalyzedCommits, + boolean exactTargetBranch) { if (unanalyzedCommits == null || unanalyzedCommits.isEmpty()) { return new CoverageResult(CoverageStatus.FULLY_COVERED, Collections.emptyList()); } // Tier 1: Check the analyzed_commit table directly (covers both branch & PR analyses) - Set alreadyRecorded = analyzedCommitRepository - .findAnalyzedHashesByProjectIdAndCommitHashIn(projectId, unanalyzedCommits); + Set alreadyRecorded = exactTargetBranch + ? analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + projectId, targetBranchName, unanalyzedCommits) + : analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndCommitHashIn( + projectId, unanalyzedCommits); List notInTable = unanalyzedCommits.stream() .filter(h -> !alreadyRecorded.contains(h)) diff --git a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java index 9472c5c83..696e056e1 100644 --- a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java +++ b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java @@ -148,7 +148,7 @@ void resolveCommitRange_normalCase_shouldResolveNewCommits() throws Exception { VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of( commit("new3"), commit("new2"), commit("new1"), commit("old-head") )); @@ -175,7 +175,7 @@ void resolveCommitRange_allCommitsAlreadyAnalyzed_shouldSkip() throws Exception VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of(commit("new1"), commit("old-head"))); when(analyzedCommitService.filterUnanalyzed(eq(1L), anyList())) @@ -199,7 +199,7 @@ void resolveCommitRange_vcsReturnsNull_shouldFallBackToHeadOnly() throws Excepti VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(null); CommitRangeContext ctx = service.resolveCommitRange(project, conn, "main", "new-head"); @@ -220,7 +220,7 @@ void resolveCommitRange_vcsReturnsEmpty_shouldFallBackToHeadOnly() throws Except VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of()); CommitRangeContext ctx = service.resolveCommitRange(project, conn, "main", "new-head"); @@ -240,7 +240,7 @@ void resolveCommitRange_lastKnownHeadNotInWindow_shouldFallBackToHeadOnly() thro VcsConnection conn = new VcsConnection(); when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); - when(vcsClient.getCommitHistory("ws", "repo", "main", 100)) + when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of(commit("c3"), commit("c2"), commit("c1"))); CommitRangeContext ctx = service.resolveCommitRange(project, conn, "main", "c3"); diff --git a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageServiceTest.java b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageServiceTest.java index 4b31362f3..c86470e30 100644 --- a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageServiceTest.java +++ b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/CommitCoverageServiceTest.java @@ -77,6 +77,25 @@ void checkCoverage_allInAnalyzedCommitTable_shouldReturnFullyCovered() { verifyNoInteractions(pullRequestRepository); // tier 2 never reached } + @Test + void checkCoverage_exactTarget_doesNotReuseDevelopReceiptForMaster() { + when(analyzedCommitRepository + .findAnalyzedHashesByProjectIdAndTargetBranchAndCommitHashIn( + 1L, "master", List.of("shared-commit"))) + .thenReturn(Set.of()); + when(pullRequestRepository.findByProjectIdAndTargetBranchNameAndStateIn( + eq(1L), eq("master"), anyList())).thenReturn(List.of()); + + CommitCoverageService.CoverageResult result = service.checkCoverage( + 1L, "master", List.of("shared-commit"), true); + + assertThat(result.status()) + .isEqualTo(CommitCoverageService.CoverageStatus.NOT_COVERED); + assertThat(result.uncoveredCommits()).containsExactly("shared-commit"); + verify(analyzedCommitRepository, never()) + .findAnalyzedHashesByProjectIdAndCommitHashIn(anyLong(), anyList()); + } + // ── Tier 2: PR coverage ────────────────────────────────────────────── @Test diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java index fa35ef06e..e3c6098d6 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java @@ -46,9 +46,39 @@ public record IssueDTO ( String issueScope, Integer endLineNumber, // Origin issue – the CodeAnalysisIssue this branch issue was cloned from (null for PR-level issues) - Long originIssueId + Long originIssueId, + // Deterministic introducing-commit provenance (distinct from PR/comment author) + String introducingCommitHash, + String introducingAuthorName, + String introducingAuthorEmail, + String authorProvenanceConfidence ) { + /** Backward-compatible constructor for callers predating SCM provenance. */ + public IssueDTO( + String id, String type, String severity, String title, + String description, String suggestedFixDescription, + String suggestedFixDiff, String file, Integer line, + Integer column, String rule, String branch, + String pullRequestId, String status, OffsetDateTime createdAt, + String issueCategory, Long analysisId, Long prNumber, + String commitHash, OffsetDateTime detectedAt, + String resolvedDescription, Long resolvedByPr, + String resolvedCommitHash, Long resolvedAnalysisId, + OffsetDateTime resolvedAt, String resolvedBy, + String vcsAuthorId, String vcsAuthorUsername, + String detectionSource, String issueScope, + Integer endLineNumber, Long originIssueId) { + this(id, type, severity, title, description, + suggestedFixDescription, suggestedFixDiff, file, line, + column, rule, branch, pullRequestId, status, createdAt, + issueCategory, analysisId, prNumber, commitHash, detectedAt, + resolvedDescription, resolvedByPr, resolvedCommitHash, + resolvedAnalysisId, resolvedAt, resolvedBy, vcsAuthorId, + vcsAuthorUsername, detectionSource, issueScope, + endLineNumber, originIssueId, null, null, null, null); + } + /** * Create an IssueDTO from an independent {@link BranchIssue}. * Reads all data from BranchIssue's own fields — never dereferences to CodeAnalysisIssue. @@ -108,7 +138,11 @@ public static IssueDTO fromBranchIssue(BranchIssue bi) { bi.getIssueScope() != null ? bi.getIssueScope().name() : null, bi.getCurrentEndLineNumber() != null ? bi.getCurrentEndLineNumber() : bi.getEndLineNumber(), - bi.getOriginIssue() != null ? bi.getOriginIssue().getId() : null + bi.getOriginIssue() != null ? bi.getOriginIssue().getId() : null, + bi.getIntroducingCommitHash(), + bi.getIntroducingAuthorName(), + bi.getIntroducingAuthorEmail(), + bi.getAuthorProvenanceConfidence() ); } @@ -162,7 +196,11 @@ public static IssueDTO fromEntity(CodeAnalysisIssue issue) { issue.getDetectionSource() != null ? issue.getDetectionSource().name() : null, issue.getIssueScope() != null ? issue.getIssueScope().name() : null, issue.getEndLineNumber(), - null // CodeAnalysisIssue has no origin + null, // CodeAnalysisIssue has no origin + issue.getIntroducingCommitHash(), + issue.getIntroducingAuthorName(), + issue.getIntroducingAuthorEmail(), + issue.getAuthorProvenanceConfidence() ); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java index 20d54bfa1..8854b7578 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java @@ -118,7 +118,9 @@ public static ProjectDTO fromProject(Project project) { rc.includePatterns(), rc.excludePatterns(), rc.multiBranchEnabled(), - rc.branchRetentionDays()); + rc.branchRetentionDays(), + rc.indexedBranches(), + rc.transientBranchIndexesEnabled()); } if (config.prAnalysisEnabled() != null) { prAnalysisEnabled = config.prAnalysisEnabled(); @@ -212,13 +214,26 @@ public record RagConfigDTO( java.util.List includePatterns, java.util.List excludePatterns, Boolean multiBranchEnabled, - Integer branchRetentionDays) { + Integer branchRetentionDays, + java.util.List indexedBranches, + Boolean transientBranchIndexesEnabled) { + public RagConfigDTO( + boolean enabled, + String branch, + java.util.List includePatterns, + java.util.List excludePatterns, + Boolean multiBranchEnabled, + Integer branchRetentionDays) { + this(enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, + branchRetentionDays, null, null); + } + /** * Backward-compatible constructor without include patterns and multi-branch * fields. */ public RagConfigDTO(boolean enabled, String branch, java.util.List excludePatterns) { - this(enabled, branch, null, excludePatterns, null, null); + this(enabled, branch, null, excludePatterns, null, null, null, null); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java index 459972134..21bbcb138 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java @@ -136,6 +136,18 @@ public class BranchIssue implements ReconcilableIssue { @Column(name = "vcs_author_username", length = 100) private String vcsAuthorUsername; + @Column(name = "introducing_commit_hash", length = 64) + private String introducingCommitHash; + + @Column(name = "introducing_author_name", length = 200) + private String introducingAuthorName; + + @Column(name = "introducing_author_email", length = 320) + private String introducingAuthorEmail; + + @Column(name = "author_provenance_confidence", length = 32) + private String authorProvenanceConfidence; + // ── Content-based tracking fields ─────────────────────────────────── /** MD5 hex of the whitespace-normalized source line at detection time. */ @@ -242,6 +254,10 @@ public static BranchIssue fromCodeAnalysisIssue(CodeAnalysisIssue cai, Branch br // VCS author bi.setVcsAuthorId(cai.getVcsAuthorId()); bi.setVcsAuthorUsername(cai.getVcsAuthorUsername()); + bi.setIntroducingCommitHash(cai.getIntroducingCommitHash()); + bi.setIntroducingAuthorName(cai.getIntroducingAuthorName()); + bi.setIntroducingAuthorEmail(cai.getIntroducingAuthorEmail()); + bi.setAuthorProvenanceConfidence(cai.getAuthorProvenanceConfidence()); // Tracking hashes bi.setLineHash(cai.getLineHash()); @@ -387,6 +403,15 @@ public void setResolved(boolean resolved) { public String getVcsAuthorUsername() { return vcsAuthorUsername; } public void setVcsAuthorUsername(String vcsAuthorUsername) { this.vcsAuthorUsername = vcsAuthorUsername; } + public String getIntroducingCommitHash() { return introducingCommitHash; } + public void setIntroducingCommitHash(String introducingCommitHash) { this.introducingCommitHash = introducingCommitHash; } + public String getIntroducingAuthorName() { return introducingAuthorName; } + public void setIntroducingAuthorName(String introducingAuthorName) { this.introducingAuthorName = introducingAuthorName; } + public String getIntroducingAuthorEmail() { return introducingAuthorEmail; } + public void setIntroducingAuthorEmail(String introducingAuthorEmail) { this.introducingAuthorEmail = introducingAuthorEmail; } + public String getAuthorProvenanceConfidence() { return authorProvenanceConfidence; } + public void setAuthorProvenanceConfidence(String authorProvenanceConfidence) { this.authorProvenanceConfidence = authorProvenanceConfidence; } + public String getLineHash() { return lineHash; } public void setLineHash(String lineHash) { this.lineHash = lineHash; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java index 577ad0d9f..e161d6771 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java @@ -87,6 +87,18 @@ public class CodeAnalysisIssue implements ReconcilableIssue { @Column(name = "vcs_author_username", length = 100) private String vcsAuthorUsername; + @Column(name = "introducing_commit_hash", length = 64) + private String introducingCommitHash; + + @Column(name = "introducing_author_name", length = 200) + private String introducingAuthorName; + + @Column(name = "introducing_author_email", length = 320) + private String introducingAuthorEmail; + + @Column(name = "author_provenance_confidence", length = 32) + private String authorProvenanceConfidence; + // --- Content-based tracking fields --- /** MD5 hex of the whitespace-normalized source line at detection time. */ @@ -213,6 +225,15 @@ public class CodeAnalysisIssue implements ReconcilableIssue { public String getVcsAuthorUsername() { return vcsAuthorUsername; } public void setVcsAuthorUsername(String vcsAuthorUsername) { this.vcsAuthorUsername = vcsAuthorUsername; } + public String getIntroducingCommitHash() { return introducingCommitHash; } + public void setIntroducingCommitHash(String introducingCommitHash) { this.introducingCommitHash = introducingCommitHash; } + public String getIntroducingAuthorName() { return introducingAuthorName; } + public void setIntroducingAuthorName(String introducingAuthorName) { this.introducingAuthorName = introducingAuthorName; } + public String getIntroducingAuthorEmail() { return introducingAuthorEmail; } + public void setIntroducingAuthorEmail(String introducingAuthorEmail) { this.introducingAuthorEmail = introducingAuthorEmail; } + public String getAuthorProvenanceConfidence() { return authorProvenanceConfidence; } + public void setAuthorProvenanceConfidence(String authorProvenanceConfidence) { this.authorProvenanceConfidence = authorProvenanceConfidence; } + public String getLineHash() { return lineHash; } public void setLineHash(String lineHash) { this.lineHash = lineHash; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java index 7af91c0fb..08bc14094 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java @@ -278,7 +278,9 @@ public void setMainBranch(String mainBranch) { this.ragConfig.includePatterns(), this.ragConfig.excludePatterns(), this.ragConfig.multiBranchEnabled(), - this.ragConfig.branchRetentionDays()); + this.ragConfig.branchRetentionDays(), + this.ragConfig.indexedBranches(), + this.ragConfig.transientBranchIndexesEnabled()); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java index 014d69fcf..43132e6e2 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.core.model.project.config; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -18,6 +19,10 @@ * be retained. PR retrieval still selects only the immutable VCS target branch; * source changes come from the exact PR overlay rather than a second branch. * - branchRetentionDays: how long to keep branch index metadata before auto-cleanup (default: 90 days) + * - indexedBranches: explicit non-primary branches whose complete snapshots are retained. + * A null/empty value preserves the legacy branchPushPatterns interpretation. + * - transientBranchIndexesEnabled: whether an analyzed PR target that is not retained + * may receive a revision-pinned temporary snapshot. */ @JsonIgnoreProperties(ignoreUnknown = true) public record RagConfig( @@ -26,33 +31,51 @@ public record RagConfig( @JsonProperty("includePatterns") List includePatterns, @JsonProperty("excludePatterns") List excludePatterns, @JsonProperty("multiBranchEnabled") Boolean multiBranchEnabled, - @JsonProperty("branchRetentionDays") Integer branchRetentionDays + @JsonProperty("branchRetentionDays") Integer branchRetentionDays, + @JsonProperty("indexedBranches") List indexedBranches, + @JsonProperty("transientBranchIndexesEnabled") Boolean transientBranchIndexesEnabled ) { public static final int DEFAULT_BRANCH_RETENTION_DAYS = 90; public RagConfig() { - this(false, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(false, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); } public RagConfig(boolean enabled) { - this(enabled, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(enabled, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); } public RagConfig(boolean enabled, String branch) { - this(enabled, branch, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(enabled, branch, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); } public RagConfig(boolean enabled, String branch, List excludePatterns) { - this(enabled, branch, null, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(enabled, branch, null, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); } public RagConfig(boolean enabled, String branch, List includePatterns, List excludePatterns) { - this(enabled, branch, includePatterns, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS); + this(enabled, branch, includePatterns, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); + } + + /** + * Backward-compatible constructor for configurations written before explicit + * retained and transient branch ownership was introduced. + */ + public RagConfig( + boolean enabled, + String branch, + List includePatterns, + List excludePatterns, + Boolean multiBranchEnabled, + Integer branchRetentionDays) { + this(enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, + branchRetentionDays, null, false); } /** * Check if multi-branch context is enabled for PR analysis. */ + @JsonIgnore public boolean isMultiBranchEnabled() { return multiBranchEnabled != null && multiBranchEnabled; } @@ -60,9 +83,32 @@ public boolean isMultiBranchEnabled() { /** * Get effective branch retention days. */ + @JsonIgnore public int getEffectiveBranchRetentionDays() { return branchRetentionDays != null ? branchRetentionDays : DEFAULT_BRANCH_RETENTION_DAYS; } + + public boolean hasExplicitIndexedBranches() { + return indexedBranches != null && indexedBranches.stream() + .anyMatch(value -> value != null && !value.isBlank()); + } + + @JsonIgnore + public List getEffectiveIndexedBranches() { + if (indexedBranches == null) { + return List.of(); + } + return indexedBranches.stream() + .filter(value -> value != null && !value.isBlank()) + .map(String::trim) + .distinct() + .toList(); + } + + @JsonIgnore + public boolean isTransientBranchIndexesEnabled() { + return isMultiBranchEnabled() && Boolean.TRUE.equals(transientBranchIndexesEnabled); + } /** * Check if a branch should have indexed context based on branchPushPatterns. @@ -71,7 +117,13 @@ public int getEffectiveBranchRetentionDays() { * @return true if branch matches any pattern and multi-branch is enabled */ public boolean shouldHaveBranchIndex(String branchName, List branchPushPatterns) { - if (!isMultiBranchEnabled() || branchPushPatterns == null || branchPushPatterns.isEmpty()) { + if (!isMultiBranchEnabled() || branchName == null || branchName.isBlank()) { + return false; + } + if (hasExplicitIndexedBranches()) { + return getEffectiveIndexedBranches().contains(branchName.trim()); + } + if (branchPushPatterns == null || branchPushPatterns.isEmpty()) { return false; } return branchPushPatterns.stream() diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java index 8cea63078..89c5b5c96 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java @@ -48,6 +48,27 @@ public class RagBranchIndex { @Column(name = "commit_hash", length = 64) private String commitHash; + @Enumerated(EnumType.STRING) + @Column(name = "index_kind", nullable = false, length = 24) + private RagBranchIndexKind indexKind = RagBranchIndexKind.LEGACY; + + @Enumerated(EnumType.STRING) + @Column(name = "lifecycle_status", nullable = false, length = 24) + private RagBranchIndexLifecycleStatus lifecycleStatus = RagBranchIndexLifecycleStatus.READY; + + @Column(name = "desired_commit_hash", length = 64) + private String desiredCommitHash; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "active_generation_id") + private RagBranchIndexGeneration activeGeneration; + + @Column(name = "last_accessed_at") + private OffsetDateTime lastAccessedAt; + + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + /** * Files that were deleted in this branch (for query-time filtering). * These files should be excluded when querying the branch's context. @@ -82,6 +103,39 @@ public RagBranchIndex(Project project, String branchName) { this.branchName = branchName; } + public RagBranchIndex(Project project, String branchName, RagBranchIndexKind indexKind) { + this(project, branchName); + this.indexKind = indexKind; + this.lifecycleStatus = RagBranchIndexLifecycleStatus.PENDING; + } + + public void requestRevision(String desiredCommitHash) { + this.desiredCommitHash = desiredCommitHash; + this.lifecycleStatus = RagBranchIndexLifecycleStatus.BUILDING; + this.errorMessage = null; + } + + public void activate(RagBranchIndexGeneration generation) { + this.activeGeneration = generation; + this.commitHash = generation.getRevision(); + this.desiredCommitHash = generation.getRevision(); + this.chunkCount = generation.getChunkCount(); + this.lifecycleStatus = RagBranchIndexLifecycleStatus.READY; + this.errorMessage = null; + this.lastAccessedAt = OffsetDateTime.now(); + } + + public void failUpdate(String errorMessage) { + this.lifecycleStatus = activeGeneration == null + ? RagBranchIndexLifecycleStatus.FAILED + : RagBranchIndexLifecycleStatus.READY; + this.errorMessage = errorMessage; + } + + public void markAccessed() { + this.lastAccessedAt = OffsetDateTime.now(); + } + public Long getId() { return id; } @@ -114,6 +168,54 @@ public void setCommitHash(String commitHash) { this.commitHash = commitHash; } + public RagBranchIndexKind getIndexKind() { + return indexKind; + } + + public void setIndexKind(RagBranchIndexKind indexKind) { + this.indexKind = indexKind; + } + + public RagBranchIndexLifecycleStatus getLifecycleStatus() { + return lifecycleStatus; + } + + public void setLifecycleStatus(RagBranchIndexLifecycleStatus lifecycleStatus) { + this.lifecycleStatus = lifecycleStatus; + } + + public String getDesiredCommitHash() { + return desiredCommitHash; + } + + public void setDesiredCommitHash(String desiredCommitHash) { + this.desiredCommitHash = desiredCommitHash; + } + + public RagBranchIndexGeneration getActiveGeneration() { + return activeGeneration; + } + + public void setActiveGeneration(RagBranchIndexGeneration activeGeneration) { + this.activeGeneration = activeGeneration; + } + + public OffsetDateTime getLastAccessedAt() { + return lastAccessedAt; + } + + public void setLastAccessedAt(OffsetDateTime lastAccessedAt) { + this.lastAccessedAt = lastAccessedAt; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } + public Set getDeletedFiles() { return deletedFiles; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGeneration.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGeneration.java new file mode 100644 index 000000000..9bb40f90b --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGeneration.java @@ -0,0 +1,149 @@ +package org.rostilos.codecrow.core.model.rag; + +import jakarta.persistence.*; + +import java.time.OffsetDateTime; + +/** + * One immutable physical representation of a branch at an exact repository + * revision. A generation is published by pointing its owning branch index at it; + * failed or incomplete generations never replace the active generation. + */ +@Entity +@Table(name = "rag_branch_index_generation", indexes = { + @Index(name = "idx_rag_branch_generation_revision", columnList = "branch_index_id, revision"), + @Index(name = "idx_rag_branch_generation_status", columnList = "branch_index_id, status") +}) +public class RagBranchIndexGeneration { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "branch_index_id", nullable = false) + private RagBranchIndex branchIndex; + + @Column(name = "revision", nullable = false, length = 64, updatable = false) + private String revision; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "parent_generation_id", updatable = false) + private RagBranchIndexGeneration parentGeneration; + + @Column(name = "seed_revision", length = 64, updatable = false) + private String seedRevision; + + @Column(name = "collection_name", nullable = false, length = 300, updatable = false) + private String collectionName; + + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, length = 24) + private RagBranchIndexGenerationStatus status = RagBranchIndexGenerationStatus.BUILDING; + + @Column(name = "manifest_digest", length = 128) + private String manifestDigest; + + @Column(name = "representation_fingerprint", length = 128, updatable = false) + private String representationFingerprint; + + @Column(name = "file_count") + private Integer fileCount; + + @Column(name = "chunk_count") + private Integer chunkCount; + + @Column(name = "created_at", nullable = false, updatable = false) + private OffsetDateTime createdAt = OffsetDateTime.now(); + + @Column(name = "activated_at") + private OffsetDateTime activatedAt; + + @Column(name = "superseded_at") + private OffsetDateTime supersededAt; + + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + public RagBranchIndexGeneration() { + } + + public RagBranchIndexGeneration( + RagBranchIndex branchIndex, + String revision, + String collectionName, + RagBranchIndexGeneration parentGeneration, + String seedRevision, + String representationFingerprint) { + this.branchIndex = branchIndex; + this.revision = revision; + this.collectionName = collectionName; + this.parentGeneration = parentGeneration; + this.seedRevision = seedRevision; + this.representationFingerprint = representationFingerprint; + } + + public void activate(String manifestDigest, int fileCount, int chunkCount) { + this.manifestDigest = manifestDigest; + this.fileCount = fileCount; + this.chunkCount = chunkCount; + this.status = RagBranchIndexGenerationStatus.ACTIVE; + this.activatedAt = OffsetDateTime.now(); + this.errorMessage = null; + } + + public void supersede() { + this.status = RagBranchIndexGenerationStatus.SUPERSEDED; + this.supersededAt = OffsetDateTime.now(); + } + + public void fail(String errorMessage) { + this.status = RagBranchIndexGenerationStatus.FAILED; + this.errorMessage = errorMessage; + } + + /** Reopen an unpublished failed generation for an idempotent retry. */ + public void retry() { + if (status != RagBranchIndexGenerationStatus.FAILED) { + throw new IllegalStateException("Only a failed generation can be retried"); + } + status = RagBranchIndexGenerationStatus.BUILDING; + errorMessage = null; + activatedAt = null; + supersededAt = null; + manifestDigest = null; + fileCount = null; + chunkCount = null; + } + + public Long getId() { return id; } + public void setId(Long id) { this.id = id; } + public RagBranchIndex getBranchIndex() { return branchIndex; } + public void setBranchIndex(RagBranchIndex branchIndex) { this.branchIndex = branchIndex; } + public String getRevision() { return revision; } + public void setRevision(String revision) { this.revision = revision; } + public RagBranchIndexGeneration getParentGeneration() { return parentGeneration; } + public void setParentGeneration(RagBranchIndexGeneration parentGeneration) { this.parentGeneration = parentGeneration; } + public String getSeedRevision() { return seedRevision; } + public void setSeedRevision(String seedRevision) { this.seedRevision = seedRevision; } + public String getCollectionName() { return collectionName; } + public void setCollectionName(String collectionName) { this.collectionName = collectionName; } + public RagBranchIndexGenerationStatus getStatus() { return status; } + public void setStatus(RagBranchIndexGenerationStatus status) { this.status = status; } + public String getManifestDigest() { return manifestDigest; } + public void setManifestDigest(String manifestDigest) { this.manifestDigest = manifestDigest; } + public String getRepresentationFingerprint() { return representationFingerprint; } + public void setRepresentationFingerprint(String representationFingerprint) { this.representationFingerprint = representationFingerprint; } + public Integer getFileCount() { return fileCount; } + public void setFileCount(Integer fileCount) { this.fileCount = fileCount; } + public Integer getChunkCount() { return chunkCount; } + public void setChunkCount(Integer chunkCount) { this.chunkCount = chunkCount; } + public OffsetDateTime getCreatedAt() { return createdAt; } + public void setCreatedAt(OffsetDateTime createdAt) { this.createdAt = createdAt; } + public OffsetDateTime getActivatedAt() { return activatedAt; } + public void setActivatedAt(OffsetDateTime activatedAt) { this.activatedAt = activatedAt; } + public OffsetDateTime getSupersededAt() { return supersededAt; } + public void setSupersededAt(OffsetDateTime supersededAt) { this.supersededAt = supersededAt; } + public String getErrorMessage() { return errorMessage; } + public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationStatus.java new file mode 100644 index 000000000..0545531be --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationStatus.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.model.rag; + +/** Immutable generation publication state. */ +public enum RagBranchIndexGenerationStatus { + BUILDING, + ACTIVE, + SUPERSEDED, + FAILED +} + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java new file mode 100644 index 000000000..3c6528449 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.model.rag; + +/** Ownership and retention policy for a branch index. */ +public enum RagBranchIndexKind { + PRIMARY, + DURABLE, + TRANSIENT, + LEGACY +} + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexLifecycleStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexLifecycleStatus.java new file mode 100644 index 000000000..9053a65d8 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexLifecycleStatus.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.model.rag; + +/** Branch-level readiness independent from any individual build attempt. */ +public enum RagBranchIndexLifecycleStatus { + PENDING, + BUILDING, + READY, + FAILED +} + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java new file mode 100644 index 000000000..120b59c1b --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperation.java @@ -0,0 +1,137 @@ +package org.rostilos.codecrow.core.model.rag; + +import jakarta.persistence.*; +import org.rostilos.codecrow.core.model.project.Project; + +import java.time.OffsetDateTime; + +/** Durable, idempotent request to move one branch index to a desired revision. */ +@Entity +@Table(name = "rag_index_operation", uniqueConstraints = { + @UniqueConstraint(name = "uq_rag_index_operation_key", columnNames = {"project_id", "operation_key"}) +}) +public class RagIndexOperation { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "project_id", nullable = false) + private Project project; + + @Column(name = "branch_name", nullable = false, length = 256) + private String branchName; + + @Column(name = "from_revision", length = 64) + private String fromRevision; + + @Column(name = "to_revision", nullable = false, length = 64) + private String toRevision; + + @Column(name = "operation_key", nullable = false, length = 128) + private String operationKey; + + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, length = 24) + private RagIndexOperationStatus status = RagIndexOperationStatus.PENDING; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "generation_id") + private RagBranchIndexGeneration generation; + + @Column(name = "job_id") + private Long jobId; + + @Column(name = "attempt_count", nullable = false) + private int attemptCount; + + @Column(name = "created_at", nullable = false, updatable = false) + private OffsetDateTime createdAt = OffsetDateTime.now(); + + @Column(name = "updated_at", nullable = false) + private OffsetDateTime updatedAt = OffsetDateTime.now(); + + @Column(name = "completed_at") + private OffsetDateTime completedAt; + + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + public RagIndexOperation() { + } + + public RagIndexOperation( + Project project, + String branchName, + String fromRevision, + String toRevision, + String operationKey) { + this.project = project; + this.branchName = branchName; + this.fromRevision = fromRevision; + this.toRevision = toRevision; + this.operationKey = operationKey; + } + + @PreUpdate + void onUpdate() { + updatedAt = OffsetDateTime.now(); + } + + public void start() { + status = RagIndexOperationStatus.RUNNING; + attemptCount++; + errorMessage = null; + completedAt = null; + } + + public void succeed(RagBranchIndexGeneration generation) { + this.generation = generation; + status = RagIndexOperationStatus.SUCCEEDED; + completedAt = OffsetDateTime.now(); + errorMessage = null; + } + + public void fail(String errorMessage) { + status = RagIndexOperationStatus.FAILED; + completedAt = OffsetDateTime.now(); + this.errorMessage = errorMessage; + } + + public void heartbeat() { + if (status == RagIndexOperationStatus.PENDING + || status == RagIndexOperationStatus.RUNNING) { + updatedAt = OffsetDateTime.now(); + } + } + + public Long getId() { return id; } + public void setId(Long id) { this.id = id; } + public Project getProject() { return project; } + public void setProject(Project project) { this.project = project; } + public String getBranchName() { return branchName; } + public void setBranchName(String branchName) { this.branchName = branchName; } + public String getFromRevision() { return fromRevision; } + public void setFromRevision(String fromRevision) { this.fromRevision = fromRevision; } + public String getToRevision() { return toRevision; } + public void setToRevision(String toRevision) { this.toRevision = toRevision; } + public String getOperationKey() { return operationKey; } + public void setOperationKey(String operationKey) { this.operationKey = operationKey; } + public RagIndexOperationStatus getStatus() { return status; } + public void setStatus(RagIndexOperationStatus status) { this.status = status; } + public RagBranchIndexGeneration getGeneration() { return generation; } + public void setGeneration(RagBranchIndexGeneration generation) { this.generation = generation; } + public Long getJobId() { return jobId; } + public void setJobId(Long jobId) { this.jobId = jobId; } + public int getAttemptCount() { return attemptCount; } + public void setAttemptCount(int attemptCount) { this.attemptCount = attemptCount; } + public OffsetDateTime getCreatedAt() { return createdAt; } + public void setCreatedAt(OffsetDateTime createdAt) { this.createdAt = createdAt; } + public OffsetDateTime getUpdatedAt() { return updatedAt; } + public void setUpdatedAt(OffsetDateTime updatedAt) { this.updatedAt = updatedAt; } + public OffsetDateTime getCompletedAt() { return completedAt; } + public void setCompletedAt(OffsetDateTime completedAt) { this.completedAt = completedAt; } + public String getErrorMessage() { return errorMessage; } + public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperationStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperationStatus.java new file mode 100644 index 000000000..42500fa3c --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagIndexOperationStatus.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.core.model.rag; + +/** Durable branch-index operation state used for restart recovery. */ +public enum RagIndexOperationStatus { + PENDING, + RUNNING, + SUCCEEDED, + FAILED +} + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java new file mode 100644 index 000000000..a73470326 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java @@ -0,0 +1,32 @@ +package org.rostilos.codecrow.core.persistence.repository.rag; + +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +@Repository +public interface RagBranchIndexGenerationRepository extends JpaRepository { + + Optional findFirstByBranchIndexIdAndRevisionAndStatusInOrderByCreatedAtDesc( + Long branchIndexId, + String revision, + List statuses); + + List findByBranchIndexIdOrderByCreatedAtDesc(Long branchIndexId); + + @Query("SELECT g FROM RagBranchIndexGeneration g " + + "JOIN g.branchIndex b WHERE b.project.id = :projectId " + + "AND b.branchName = :branchName AND g.revision = :revision " + + "AND g.status IN :statuses ORDER BY g.createdAt DESC") + List findAvailableExactGeneration( + @Param("projectId") Long projectId, + @Param("branchName") String branchName, + @Param("revision") String revision, + @Param("statuses") List statuses); +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java index 09743ea07..7fa4cd5d4 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java @@ -1,6 +1,7 @@ package org.rostilos.codecrow.core.persistence.repository.rag; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -20,6 +21,8 @@ public interface RagBranchIndexRepository extends JpaRepository findByProjectId(Long projectId); + List findByIndexKind(RagBranchIndexKind indexKind); + @Query("SELECT CASE WHEN COUNT(b) > 0 THEN true ELSE false END FROM RagBranchIndex b " + "WHERE b.project.id = :projectId AND b.branchName = :branchName") boolean existsByProjectIdAndBranchName(@Param("projectId") Long projectId, @Param("branchName") String branchName); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java new file mode 100644 index 000000000..92d4b4c14 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagIndexOperationRepository.java @@ -0,0 +1,20 @@ +package org.rostilos.codecrow.core.persistence.repository.rag; + +import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +@Repository +public interface RagIndexOperationRepository extends JpaRepository { + + Optional findByProjectIdAndOperationKey(Long projectId, String operationKey); + + List findByStatusInAndUpdatedAtBefore( + List statuses, + OffsetDateTime updatedBefore); +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java index aea9361bc..414ad18f9 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java @@ -519,6 +519,10 @@ public CodeAnalysis cloneAnalysisForPr( issueClone.setResolvedDescription(srcIssue.getResolvedDescription()); issueClone.setVcsAuthorId(srcIssue.getVcsAuthorId()); issueClone.setVcsAuthorUsername(srcIssue.getVcsAuthorUsername()); + issueClone.setIntroducingCommitHash(srcIssue.getIntroducingCommitHash()); + issueClone.setIntroducingAuthorName(srcIssue.getIntroducingAuthorName()); + issueClone.setIntroducingAuthorEmail(srcIssue.getIntroducingAuthorEmail()); + issueClone.setAuthorProvenanceConfidence(srcIssue.getAuthorProvenanceConfidence()); // Copy content-based tracking hashes issueClone.setLineHash(srcIssue.getLineHash()); issueClone.setLineHashContext(srcIssue.getLineHashContext()); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.23.0__exact_branch_index_registry.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.23.0__exact_branch_index_registry.sql new file mode 100644 index 000000000..f21e78f41 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.23.0__exact_branch_index_registry.sql @@ -0,0 +1,126 @@ +-- Establish durable ownership and immutable generation state for exact branch indexes. +-- The early multi-branch migration dropped rag_delta_index but did not create its +-- replacement, so this forward-only migration safely handles both fresh and +-- Hibernate-created installations. + +CREATE TABLE IF NOT EXISTS rag_branch_index ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + branch_name VARCHAR(256) NOT NULL, + commit_hash VARCHAR(64), + chunk_count INTEGER, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_rag_branch_index_project_branch UNIQUE (project_id, branch_name) +); + +ALTER TABLE rag_branch_index + ADD COLUMN IF NOT EXISTS index_kind VARCHAR(24) NOT NULL DEFAULT 'LEGACY', + ADD COLUMN IF NOT EXISTS lifecycle_status VARCHAR(24) NOT NULL DEFAULT 'READY', + ADD COLUMN IF NOT EXISTS desired_commit_hash VARCHAR(64), + ADD COLUMN IF NOT EXISTS active_generation_id BIGINT, + ADD COLUMN IF NOT EXISTS last_accessed_at TIMESTAMP WITH TIME ZONE, + ADD COLUMN IF NOT EXISTS error_message TEXT; + +CREATE INDEX IF NOT EXISTS idx_rag_branch_project + ON rag_branch_index(project_id); +CREATE INDEX IF NOT EXISTS idx_rag_branch_name + ON rag_branch_index(branch_name); +CREATE INDEX IF NOT EXISTS idx_rag_branch_lifecycle + ON rag_branch_index(project_id, lifecycle_status); + +CREATE TABLE IF NOT EXISTS rag_branch_deleted_files ( + branch_index_id BIGINT NOT NULL REFERENCES rag_branch_index(id) ON DELETE CASCADE, + file_path VARCHAR(512) NOT NULL, + CONSTRAINT uq_rag_branch_deleted_file UNIQUE (branch_index_id, file_path) +); + +CREATE TABLE IF NOT EXISTS rag_branch_index_generation ( + id BIGSERIAL PRIMARY KEY, + branch_index_id BIGINT NOT NULL REFERENCES rag_branch_index(id) ON DELETE CASCADE, + revision VARCHAR(64) NOT NULL, + parent_generation_id BIGINT REFERENCES rag_branch_index_generation(id) ON DELETE SET NULL, + seed_revision VARCHAR(64), + collection_name VARCHAR(300) NOT NULL, + status VARCHAR(24) NOT NULL, + manifest_digest VARCHAR(128), + representation_fingerprint VARCHAR(128), + file_count INTEGER, + chunk_count INTEGER, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + activated_at TIMESTAMP WITH TIME ZONE, + superseded_at TIMESTAMP WITH TIME ZONE, + error_message TEXT, + CONSTRAINT uq_rag_branch_generation_collection UNIQUE (branch_index_id, collection_name), + CONSTRAINT ck_rag_branch_generation_status + CHECK (status IN ('BUILDING', 'ACTIVE', 'SUPERSEDED', 'FAILED')), + CONSTRAINT ck_rag_branch_generation_counts + CHECK ((file_count IS NULL OR file_count >= 0) AND (chunk_count IS NULL OR chunk_count >= 0)) +); + +CREATE INDEX IF NOT EXISTS idx_rag_branch_generation_revision + ON rag_branch_index_generation(branch_index_id, revision); +CREATE INDEX IF NOT EXISTS idx_rag_branch_generation_status + ON rag_branch_index_generation(branch_index_id, status); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'fk_rag_branch_active_generation' + AND conrelid = 'rag_branch_index'::regclass + ) THEN + ALTER TABLE rag_branch_index + ADD CONSTRAINT fk_rag_branch_active_generation + FOREIGN KEY (active_generation_id) + REFERENCES rag_branch_index_generation(id) + ON DELETE SET NULL; + END IF; +END; +$$; + +CREATE TABLE IF NOT EXISTS rag_index_operation ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + branch_name VARCHAR(256) NOT NULL, + from_revision VARCHAR(64), + to_revision VARCHAR(64) NOT NULL, + operation_key VARCHAR(128) NOT NULL, + status VARCHAR(24) NOT NULL, + generation_id BIGINT REFERENCES rag_branch_index_generation(id) ON DELETE SET NULL, + job_id BIGINT, + attempt_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP WITH TIME ZONE, + error_message TEXT, + CONSTRAINT uq_rag_index_operation_key UNIQUE (project_id, operation_key), + CONSTRAINT ck_rag_index_operation_status + CHECK (status IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED')), + CONSTRAINT ck_rag_index_operation_attempts CHECK (attempt_count >= 0) +); + +CREATE INDEX IF NOT EXISTS idx_rag_index_operation_recovery + ON rag_index_operation(status, updated_at); +CREATE INDEX IF NOT EXISTS idx_rag_index_operation_branch + ON rag_index_operation(project_id, branch_name, to_revision); + +-- Existing rows represent the shared-collection implementation. Only the row +-- matching the authoritative RagIndexStatus branch can be identified as primary; +-- every other legacy row remains unverified until rebuilt as an exact generation. +UPDATE rag_branch_index branch_index +SET index_kind = CASE + WHEN EXISTS ( + SELECT 1 + FROM rag_index_status status + WHERE status.project_id = branch_index.project_id + AND status.indexed_branch = branch_index.branch_name + ) THEN 'PRIMARY' + ELSE 'LEGACY' + END, + lifecycle_status = CASE + WHEN branch_index.commit_hash IS NULL THEN 'PENDING' + ELSE 'READY' + END +WHERE active_generation_id IS NULL; + diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.24.0__scm_evidence_and_branch_analysis_scope.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.24.0__scm_evidence_and_branch_analysis_scope.sql new file mode 100644 index 000000000..b3a62ed06 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.24.0__scm_evidence_and_branch_analysis_scope.sql @@ -0,0 +1,76 @@ +-- Deterministic commit evidence is reusable across promotions, while analysis +-- receipts remain scoped to the exact target branch context. + +ALTER TABLE analyzed_commit + ADD COLUMN IF NOT EXISTS source_branch VARCHAR(256), + ADD COLUMN IF NOT EXISTS target_branch VARCHAR(256), + ADD COLUMN IF NOT EXISTS target_base_revision VARCHAR(64); + +ALTER TABLE analyzed_commit + DROP CONSTRAINT IF EXISTS uq_analyzed_commit_project_hash; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_analyzed_commit_project_target_hash + ON analyzed_commit(project_id, COALESCE(target_branch, ''), commit_hash); +CREATE INDEX IF NOT EXISTS idx_analyzed_commit_target + ON analyzed_commit(project_id, target_branch, commit_hash); + +CREATE TABLE scm_commit_evidence ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + commit_hash VARCHAR(64) NOT NULL, + patch_id VARCHAR(64) NOT NULL, + author_name VARCHAR(200), + author_email VARCHAR(320), + captured_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_scm_commit_evidence_project_hash + UNIQUE (project_id, commit_hash), + CONSTRAINT ck_scm_commit_patch_id CHECK (patch_id ~ '^[0-9a-f]{64}$') +); +CREATE INDEX idx_scm_commit_patch + ON scm_commit_evidence(project_id, patch_id); + +CREATE TABLE scm_added_line_evidence ( + id BIGSERIAL PRIMARY KEY, + commit_evidence_id BIGINT NOT NULL + REFERENCES scm_commit_evidence(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + file_path VARCHAR(1024) NOT NULL, + new_line_number INTEGER NOT NULL, + line_hash VARCHAR(64) NOT NULL, + CONSTRAINT ck_scm_added_line_number CHECK (new_line_number > 0), + CONSTRAINT ck_scm_added_line_hash CHECK (line_hash ~ '^[0-9a-f]{64}$') +); +CREATE INDEX idx_scm_added_line_lookup + ON scm_added_line_evidence(project_id, file_path, line_hash); + +CREATE TABLE scm_analysis_receipt ( + id BIGSERIAL PRIMARY KEY, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + commit_evidence_id BIGINT NOT NULL + REFERENCES scm_commit_evidence(id) ON DELETE CASCADE, + source_branch VARCHAR(256), + target_branch VARCHAR(256) NOT NULL, + target_base_revision VARCHAR(64), + analysis_id BIGINT, + analysis_type VARCHAR(40) NOT NULL, + context_key VARCHAR(64) NOT NULL, + analyzed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT uq_scm_analysis_receipt_context + UNIQUE (project_id, commit_evidence_id, context_key), + CONSTRAINT ck_scm_analysis_context_key + CHECK (context_key ~ '^[0-9a-f]{64}$') +); +CREATE INDEX idx_scm_analysis_receipt_patch + ON scm_analysis_receipt(project_id, commit_evidence_id, target_branch); + +ALTER TABLE code_analysis_issue + ADD COLUMN IF NOT EXISTS introducing_commit_hash VARCHAR(64), + ADD COLUMN IF NOT EXISTS introducing_author_name VARCHAR(200), + ADD COLUMN IF NOT EXISTS introducing_author_email VARCHAR(320), + ADD COLUMN IF NOT EXISTS author_provenance_confidence VARCHAR(32); + +ALTER TABLE branch_issue + ADD COLUMN IF NOT EXISTS introducing_commit_hash VARCHAR(64), + ADD COLUMN IF NOT EXISTS introducing_author_name VARCHAR(200), + ADD COLUMN IF NOT EXISTS introducing_author_email VARCHAR(320), + ADD COLUMN IF NOT EXISTS author_provenance_confidence VARCHAR(32); diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java index 1e0ccc99c..d769c9e3a 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.core.model.project.config; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import java.util.List; @@ -104,4 +105,66 @@ void shouldSupportInequality() { void shouldHaveDefaultBranchRetentionDaysConstant() { assertThat(RagConfig.DEFAULT_BRANCH_RETENTION_DAYS).isEqualTo(90); } + + @Test + void explicitIndexedBranchesShouldOverrideLegacyPushPatterns() { + RagConfig config = new RagConfig( + true, + "master", + null, + null, + true, + 30, + List.of(" develop ", "support/1.x", "develop", " "), + true); + + assertThat(config.getEffectiveIndexedBranches()).containsExactly("develop", "support/1.x"); + assertThat(config.shouldHaveBranchIndex("develop", List.of("feature/**"))).isTrue(); + assertThat(config.shouldHaveBranchIndex("feature/one", List.of("feature/**"))).isFalse(); + assertThat(config.isTransientBranchIndexesEnabled()).isTrue(); + } + + @Test + void legacyConfigurationShouldContinueUsingPushPatterns() { + RagConfig config = new RagConfig(true, "master", null, null, true, 30); + + assertThat(config.hasExplicitIndexedBranches()).isFalse(); + assertThat(config.shouldHaveBranchIndex("develop", List.of("develop", "support/**"))).isTrue(); + assertThat(config.shouldHaveBranchIndex("support/1.x", List.of("develop", "support/**"))).isTrue(); + assertThat(config.shouldHaveBranchIndex("feature/one", List.of("develop", "support/**"))).isFalse(); + assertThat(config.isTransientBranchIndexesEnabled()).isFalse(); + } + + @Test + void transientIndexesRequireMultiBranchOwnership() { + RagConfig config = new RagConfig( + true, "master", null, null, false, 30, List.of("develop"), true); + + assertThat(config.isTransientBranchIndexesEnabled()).isFalse(); + assertThat(config.shouldHaveBranchIndex("develop", List.of("develop"))).isFalse(); + } + + @Test + void serializesOnlyPersistedFieldsAndRoundTripsRetainedBranches() throws Exception { + ProjectConfig projectConfig = new ProjectConfig(); + projectConfig.setRagConfig(new RagConfig( + true, "master", null, null, true, 30, + List.of("develop", "release/1.x"), true)); + + ObjectMapper mapper = new ObjectMapper(); + String json = mapper.writeValueAsString(projectConfig); + + assertThat(json) + .doesNotContain("effectiveIndexedBranches") + .doesNotContain("effectiveBranchRetentionDays") + .doesNotContain("isMultiBranchEnabled") + .doesNotContain("isTransientBranchIndexesEnabled"); + + ProjectConfig restored = mapper.readValue(json, ProjectConfig.class); + assertThat(restored.ragConfig().indexedBranches()) + .containsExactly("develop", "release/1.x"); + assertThat(restored.ragConfig().getEffectiveIndexedBranches()) + .containsExactly("develop", "release/1.x"); + assertThat(restored.ragConfig().isTransientBranchIndexesEnabled()).isTrue(); + } } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java new file mode 100644 index 000000000..41fcb48dd --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java @@ -0,0 +1,78 @@ +package org.rostilos.codecrow.core.model.rag; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; + +import static org.assertj.core.api.Assertions.assertThat; + +class RagBranchIndexGenerationTest { + + @Test + void activatesExactGenerationAndMakesItTheBranchCheckpoint() { + RagBranchIndex branchIndex = new RagBranchIndex( + new Project(), "develop", RagBranchIndexKind.DURABLE); + branchIndex.requestRevision("develop-002"); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, + "develop-002", + "tenant-1-project-2-develop-generation-2", + null, + "master-100", + "representation-digest"); + + generation.activate("manifest-digest", 231, 840); + branchIndex.activate(generation); + + assertThat(generation.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.ACTIVE); + assertThat(generation.getActivatedAt()).isNotNull(); + assertThat(branchIndex.getActiveGeneration()).isSameAs(generation); + assertThat(branchIndex.getCommitHash()).isEqualTo("develop-002"); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-002"); + assertThat(branchIndex.getChunkCount()).isEqualTo(840); + assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); + } + + @Test + void failedReplacementPreservesPreviouslyActiveGeneration() { + RagBranchIndex branchIndex = new RagBranchIndex( + new Project(), "master", RagBranchIndexKind.PRIMARY); + RagBranchIndexGeneration active = new RagBranchIndexGeneration( + branchIndex, "master-100", "master-generation-100", null, + null, "representation-digest"); + active.activate("manifest-100", 400, 1200); + branchIndex.activate(active); + + branchIndex.requestRevision("master-101"); + RagBranchIndexGeneration replacement = new RagBranchIndexGeneration( + branchIndex, "master-101", "master-generation-101", active, + "master-100", "representation-digest"); + replacement.fail("Qdrant unavailable"); + branchIndex.failUpdate(replacement.getErrorMessage()); + + assertThat(branchIndex.getActiveGeneration()).isSameAs(active); + assertThat(branchIndex.getCommitHash()).isEqualTo("master-100"); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("master-101"); + assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); + assertThat(branchIndex.getErrorMessage()).isEqualTo("Qdrant unavailable"); + assertThat(replacement.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.FAILED); + } + + @Test + void durableOperationTracksAttemptsAndCompletion() { + Project project = new Project(); + RagIndexOperation operation = new RagIndexOperation( + project, "support/1.x", "support-9", "support-10", "operation-digest"); + RagBranchIndex branchIndex = new RagBranchIndex(project, "support/1.x", RagBranchIndexKind.DURABLE); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, "support-10", "support-generation-10", null, + "master-100", "representation-digest"); + + operation.start(); + operation.succeed(generation); + + assertThat(operation.getAttemptCount()).isEqualTo(1); + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); + assertThat(operation.getGeneration()).isSameAs(generation); + assertThat(operation.getCompletedAt()).isNotNull(); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java new file mode 100644 index 000000000..b5627652b --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java @@ -0,0 +1,28 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Executor; + +/** Dedicated bounded concurrency for independent configured branch snapshots. */ +@Configuration +public class BranchIndexBuildExecutorConfiguration { + + @Bean(name = "branchIndexBuildExecutor") + public Executor branchIndexBuildExecutor( + @Value("${codecrow.rag.branch-build.parallelism:4}") int parallelism) { + int workers = Math.max(1, parallelism); + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(workers); + executor.setMaxPoolSize(workers); + executor.setQueueCapacity(50); + executor.setThreadNamePrefix("rag-branch-build-"); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(300); + executor.initialize(); + return executor; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java new file mode 100644 index 000000000..04cbb8351 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java @@ -0,0 +1,245 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Builds one exact branch generation without depending on analysis processors. + * VCS acquisition, vector storage, and registry publication meet only through + * this small orchestration boundary. + */ +@Service +public class BranchIndexGenerationBuildService { + private static final long HEARTBEAT_INTERVAL_SECONDS = 15; + + private final BranchArchiveService archiveService; + private final RagPipelineClient pipelineClient; + private final RagBranchIndexRegistryService registryService; + private final ScheduledExecutorService heartbeatExecutor; + + public BranchIndexGenerationBuildService( + BranchArchiveService archiveService, + RagPipelineClient pipelineClient, + RagBranchIndexRegistryService registryService) { + this.archiveService = archiveService; + this.pipelineClient = pipelineClient; + this.registryService = registryService; + this.heartbeatExecutor = Executors.newSingleThreadScheduledExecutor(new HeartbeatThreadFactory()); + } + + public Map build( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns) throws IOException { + return build(project, connection, vcsWorkspace, repoSlug, branch, + revision, kind, includePatterns, excludePatterns, null); + } + + public Map build( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId, + Consumer> progressEvents) throws IOException { + return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, + revision, kind, includePatterns, excludePatterns, jobId, progressEvents, false); + } + + /** + * Builds a fresh immutable generation even when this revision already has + * a successful generation. This is reserved for an explicit operator + * refresh; automatic maintenance remains idempotent. + */ + public Map rebuild( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId, + Consumer> progressEvents) throws IOException { + return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, + revision, kind, includePatterns, excludePatterns, jobId, progressEvents, true); + } + + public Map build( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId) throws IOException { + return buildInternal(project, connection, vcsWorkspace, repoSlug, branch, + revision, kind, includePatterns, excludePatterns, jobId, null, false); + } + + private Map buildInternal( + Project project, + VcsConnection connection, + String vcsWorkspace, + String repoSlug, + String branch, + String revision, + RagBranchIndexKind kind, + List includePatterns, + List excludePatterns, + Long jobId, + Consumer> progressEvents, + boolean forceRebuild) throws IOException { + var registration = registryService.registerBuild( + project, branch, kind, null, revision, + forceRebuild ? "operator-refresh:" + requireJobId(jobId) : null); + if (registration.existingOperation() + && registration.operation().getStatus() + == RagIndexOperationStatus.SUCCEEDED) { + return Map.of( + "status", "reused", + "collection_target", registration.generation().getCollectionName(), + "generation_manifest_sha256", registration.generation().getManifestDigest()); + } + + registryService.startBuild(registration.operation().getId(), jobId); + ScheduledFuture heartbeat = heartbeatExecutor.scheduleAtFixedRate( + () -> heartbeat(registration.operation().getId()), + HEARTBEAT_INTERVAL_SECONDS, + HEARTBEAT_INTERVAL_SECONDS, + TimeUnit.SECONDS); + Path snapshot = null; + try { + snapshot = Files.createTempDirectory("codecrow-rag-branch-generation-"); + archiveService.downloadAndExtractSnapshotToDirectory( + connection, + vcsWorkspace, + repoSlug, + revision, + null, + snapshot); + boolean publishBranchAlias = kind == RagBranchIndexKind.PRIMARY + || kind == RagBranchIndexKind.DURABLE; + boolean publishLegacyProjectAlias = kind == RagBranchIndexKind.PRIMARY; + Map result = progressEvents == null + ? pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, registration.generation().getCollectionName(), + publishBranchAlias, publishLegacyProjectAlias) + : pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, registration.generation().getCollectionName(), + publishBranchAlias, publishLegacyProjectAlias, + progressEvents); + Object manifest = result.get("generation_manifest_sha256"); + if (!(manifest instanceof String digest) || digest.isBlank()) { + throw new IOException("RAG full branch generation has no manifest digest"); + } + registryService.publish( + registration.operation().getId(), + digest, + number(result.get("document_count")), + number(result.get("chunk_count"))); + return result; + } catch (Throwable failure) { + registryService.fail( + registration.operation().getId(), + failure.getMessage() != null + ? failure.getMessage() + : failure.getClass().getSimpleName()); + if (failure instanceof IOException ioFailure) { + throw ioFailure; + } + if (failure instanceof Error error) { + throw error; + } + throw new IOException("Failed to build exact branch generation", failure); + } finally { + heartbeat.cancel(false); + if (snapshot != null) { + deleteTree(snapshot); + } + } + } + + private static int number(Object value) { + return value instanceof Number number ? number.intValue() : 0; + } + + private static String requireJobId(Long jobId) { + if (jobId == null) { + throw new IllegalArgumentException("An operator refresh requires a durable job id"); + } + return jobId.toString(); + } + + private void heartbeat(long operationId) { + try { + registryService.heartbeatBuild(operationId); + } catch (Exception ignored) { + // A later heartbeat may still succeed. If the producer actually + // stops, recovery turns the durable operation into a failure. + } + } + + private static final class HeartbeatThreadFactory implements ThreadFactory { + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "rag-generation-heartbeat"); + thread.setDaemon(true); + return thread; + } + } + + private static void deleteTree(Path root) { + try { + if (!Files.exists(root)) { + return; + } + try (var paths = Files.walk(root)) { + for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } + } catch (IOException ignored) { + // Temporary cleanup must not replace the indexing result. + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java new file mode 100644 index 000000000..b8c4cee1b --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java @@ -0,0 +1,260 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.core.service.AnalysisJobService; +import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +/** + * Explicit operator-triggered rebuilds for configured RAG branches. + * + * This is deliberately separate from webhook reconciliation: it always builds + * an exact complete snapshot for the requested revision, and only accepts the + * primary branch or an explicitly retained target. Each branch build has its + * own durable operation and job, so an all-branch run is observable and safe + * to repeat after a partial failure. + */ +@Service +public class BranchIndexMaintenanceService { + private final RagOperationsService ragOperationsService; + private final VcsClientProvider vcsClientProvider; + private final BranchIndexGenerationBuildService generationBuildService; + private final RagIndexTrackingService trackingService; + private final AnalysisLockService lockService; + private final AnalysisJobService jobService; + private final Executor branchIndexBuildExecutor; + + public BranchIndexMaintenanceService( + RagOperationsService ragOperationsService, + VcsClientProvider vcsClientProvider, + BranchIndexGenerationBuildService generationBuildService, + RagIndexTrackingService trackingService, + AnalysisLockService lockService, + AnalysisJobService jobService, + @Qualifier("branchIndexBuildExecutor") Executor branchIndexBuildExecutor) { + this.ragOperationsService = ragOperationsService; + this.vcsClientProvider = vcsClientProvider; + this.generationBuildService = generationBuildService; + this.trackingService = trackingService; + this.lockService = lockService; + this.jobService = jobService; + this.branchIndexBuildExecutor = branchIndexBuildExecutor; + } + + public Map rebuild(Project project, String requestedBranch, boolean allConfiguredBranches, + Consumer> events) { + List branches = resolveBranches(project, requestedBranch, allConfiguredBranches); + List completed = new ArrayList<>(); + Map failures = new LinkedHashMap<>(); + Map> builds = new LinkedHashMap<>(); + + // Obtaining a provider client can refresh a shared installation token. Do + // that small VCS preparation phase once at a time, then let the expensive + // archive download and RAG mutation for every resolved branch run in + // parallel. Concurrent token refreshes previously allowed one branch to + // disappear before it had a durable job or operation to report. + List plans = new ArrayList<>(); + for (String branch : branches) { + try { + plans.add(prepareBuild(project, branch)); + } catch (RuntimeException failure) { + String message = failure.getMessage() != null + ? failure.getMessage() : failure.getClass().getSimpleName(); + failures.put(branch, message); + events.accept(Map.of("type", "progress", "stage", "branch_failed", + "branch", branch, + "message", "RAG snapshot failed for branch '" + branch + "': " + message)); + } + } + for (BranchBuildPlan plan : plans) { + String branch = plan.branch(); + builds.put(branch, CompletableFuture.runAsync(() -> { + events.accept(Map.of("type", "progress", "stage", "branch", + "branch", branch, + "message", "Building exact RAG snapshot for branch '" + branch + "'")); + rebuildOne(project, plan, events); + }, branchIndexBuildExecutor)); + } + for (Map.Entry> build : builds.entrySet()) { + try { + build.getValue().join(); + completed.add(build.getKey()); + } catch (CompletionException failure) { + Throwable cause = failure.getCause() != null ? failure.getCause() : failure; + String message = cause.getMessage() != null + ? cause.getMessage() : cause.getClass().getSimpleName(); + failures.put(build.getKey(), message); + events.accept(Map.of("type", "progress", "stage", "branch_failed", + "branch", build.getKey(), + "message", "RAG snapshot failed for branch '" + build.getKey() + + "': " + message)); + } + } + if (completed.isEmpty()) { + throw new IllegalStateException("No configured branch snapshot was built: " + failures); + } + Map outcome = new LinkedHashMap<>(); + outcome.put("status", "completed"); + outcome.put("message", failures.isEmpty() + ? "Built RAG snapshots for " + String.join(", ", completed) + : "Built RAG snapshots for " + String.join(", ", completed) + + "; failed: " + String.join(", ", failures.keySet())); + outcome.put("branches", completed); + outcome.put("failedBranches", failures); + return outcome; + } + + private BranchBuildPlan prepareBuild(Project project, String branch) { + var binding = project.getVcsRepoBinding(); + if (binding == null || binding.getVcsConnection() == null) { + throw new IllegalStateException("Project has no VCS connection"); + } + VcsConnection connection = binding.getVcsConnection(); + String workspace = binding.getExternalNamespace(); + String repository = binding.getExternalRepoSlug(); + VcsClient client = vcsClientProvider.getClient(connection); + String revision; + try { + revision = client.getLatestCommitHash(workspace, repository, branch); + } catch (Exception failure) { + throw new IllegalStateException("Could not resolve the current revision for branch '" + branch + "'", failure); + } + if (revision == null || revision.isBlank()) { + throw new IllegalStateException("Branch '" + branch + "' has no resolvable revision"); + } + return new BranchBuildPlan(branch, connection, workspace, repository, revision); + } + + private void rebuildOne(Project project, BranchBuildPlan plan, Consumer> events) { + String branch = plan.branch(); + VcsConnection connection = plan.connection(); + String workspace = plan.workspace(); + String repository = plan.repository(); + String revision = plan.revision(); + + Optional lock = lockService.acquireLock( + project, branch, AnalysisLockType.RAG_INDEXING, revision, null); + if (lock.isEmpty()) { + throw new IllegalStateException("RAG indexing is already running for branch '" + branch + "'"); + } + + boolean primary = branch.equals(ragOperationsService.getBaseBranch(project)); + boolean primaryPreviouslyIndexed = primary && trackingService.isProjectIndexed(project); + Job job = jobService.createRagIndexJob(project, !primaryPreviouslyIndexed, JobTriggerSource.UI); + jobService.startJob(job); + jobService.info(job, "branch_snapshot", "Building exact RAG snapshot for branch: " + branch); + try { + if (primary) { + trackingService.markIndexingStarted(project, branch, revision); + } + var config = project.getConfiguration().ragConfig(); + Map result = generationBuildService.rebuild( + project, + connection, + workspace, + repository, + branch, + revision, + primary ? RagBranchIndexKind.PRIMARY : RagBranchIndexKind.DURABLE, + config.includePatterns(), + config.excludePatterns(), + job.getId(), event -> { + Map forwarded = new LinkedHashMap<>(event); + forwarded.put("type", "progress"); + forwarded.put("branch", branch); + events.accept(forwarded); + }); + if (primary) { + trackingService.markIndexingCompleted( + project, + branch, + revision, + number(result.get("document_count")), + number(result.get("chunk_count"))); + } + jobService.completeJob(job, Map.of("branch", branch, "revision", revision)); + events.accept(Map.of("type", "progress", "stage", "branch_complete", "branch", branch, + "message", "RAG snapshot is ready for branch '" + branch + "'")); + } catch (Throwable failure) { + String diagnostic = failure.getMessage() != null + ? failure.getMessage() : failure.getClass().getSimpleName(); + if (primary) { + if (primaryPreviouslyIndexed) { + trackingService.markIncrementalUpdateFailed(project, diagnostic); + } else { + trackingService.markIndexingFailed(project, diagnostic); + } + } + jobService.failJob(job, diagnostic); + if (failure instanceof Error error) { + throw error; + } + throw failure instanceof RuntimeException runtime ? runtime + : new IllegalStateException("Failed to build RAG snapshot for branch '" + branch + "'", failure); + } finally { + lockService.releaseLock(lock.get()); + } + } + + private List resolveBranches(Project project, String requestedBranch, boolean allConfiguredBranches) { + if (project.getConfiguration() == null || project.getConfiguration().ragConfig() == null + || !project.getConfiguration().ragConfig().enabled()) { + throw new IllegalStateException("RAG is not enabled for this project"); + } + var config = project.getConfiguration().ragConfig(); + String primary = ragOperationsService.getBaseBranch(project); + if (allConfiguredBranches) { + if (!config.isMultiBranchEnabled()) { + return List.of(primary); + } + LinkedHashSet branches = new LinkedHashSet<>(); + branches.add(primary); + branches.addAll(config.getEffectiveIndexedBranches()); + return List.copyOf(branches); + } + if (requestedBranch == null || requestedBranch.isBlank()) { + throw new IllegalArgumentException("A configured RAG branch must be selected"); + } + String branch = requestedBranch.trim(); + if (branch.equals(primary)) { + return List.of(branch); + } + if (config.isMultiBranchEnabled() && ragOperationsService.shouldHaveBranchIndex(project, branch)) { + return List.of(branch); + } + throw new IllegalArgumentException("Branch '" + branch + "' is not configured as a retained RAG branch"); + } + + private record BranchBuildPlan( + String branch, + VcsConnection connection, + String workspace, + String repository, + String revision) { + } + + private static int number(Object value) { + return value instanceof Number number ? number.intValue() : 0; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java new file mode 100644 index 000000000..b58b5ea5c --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java @@ -0,0 +1,74 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * Restores human-readable Qdrant aliases for already active branch generations. + * + * Normal generation activation publishes those aliases atomically alongside its + * immutable target. This independent, idempotent repair loop makes the feature + * safe for projects and Qdrant snapshots created before that contract existed. + * It never changes the registry or analysis binding, so a temporary Qdrant + * failure only affects operator discoverability and is retried later. + */ +@Service +public class RagBranchOperatorAliasReconciliationService { + private static final Logger log = LoggerFactory.getLogger( + RagBranchOperatorAliasReconciliationService.class); + + private final RagBranchIndexRepository branchIndexRepository; + private final RagPipelineClient pipelineClient; + + public RagBranchOperatorAliasReconciliationService( + RagBranchIndexRepository branchIndexRepository, + RagPipelineClient pipelineClient) { + this.branchIndexRepository = branchIndexRepository; + this.pipelineClient = pipelineClient; + } + + @Scheduled( + fixedDelayString = "${codecrow.rag.operator-alias.reconcile-interval-ms:300000}", + initialDelayString = "${codecrow.rag.operator-alias.reconcile-initial-delay-ms:15000}") + @Transactional(readOnly = true) + public void reconcileActiveGenerationAliases() { + for (RagBranchIndex branchIndex : branchIndexRepository.findAll()) { + if (!isPublishedOperatorBranch(branchIndex)) { + continue; + } + var generation = branchIndex.getActiveGeneration(); + var project = branchIndex.getProject(); + try { + pipelineClient.publishGenerationAliases( + project.getWorkspace().getName(), + project.getNamespace(), + branchIndex.getBranchName(), + generation.getRevision(), + generation.getCollectionName(), + true, + branchIndex.getIndexKind() == RagBranchIndexKind.PRIMARY); + } catch (Exception failure) { + log.warn( + "Could not reconcile readable RAG alias for project={} branch={}: {}", + project.getId(), + branchIndex.getBranchName(), + failure.getMessage()); + } + } + } + + private static boolean isPublishedOperatorBranch(RagBranchIndex branchIndex) { + if (branchIndex == null || branchIndex.getActiveGeneration() == null) { + return false; + } + return branchIndex.getIndexKind() == RagBranchIndexKind.PRIMARY + || branchIndex.getIndexKind() == RagBranchIndexKind.DURABLE; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java new file mode 100644 index 000000000..7c1945282 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java @@ -0,0 +1,52 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; + +/** + * Terminates registry operations whose producer disappeared before publishing + * a generation. The prior active generation remains queryable; a later event + * retries the same idempotency key from a fresh snapshot or checkpoint. + */ +@Service +public class RagIndexOperationRecoveryService { + + private static final Logger log = LoggerFactory.getLogger( + RagIndexOperationRecoveryService.class); + + private final RagBranchIndexRegistryService registryService; + private final long staleAfterMinutes; + + public RagIndexOperationRecoveryService( + RagBranchIndexRegistryService registryService, + @Value("${codecrow.rag.generation.stale-after-minutes:30}") + long staleAfterMinutes) { + this.registryService = registryService; + this.staleAfterMinutes = Math.max(5, staleAfterMinutes); + } + + @Scheduled( + fixedDelayString = "${codecrow.rag.generation.recovery-interval-ms:300000}", + initialDelayString = "${codecrow.rag.generation.recovery-initial-delay-ms:60000}") + public void failAbandonedOperations() { + OffsetDateTime cutoff = OffsetDateTime.now().minusMinutes(staleAfterMinutes); + for (var operation : registryService.findRecoverableOperations(cutoff)) { + String diagnostic = "Exact RAG generation producer stopped heartbeating for " + + staleAfterMinutes + " minutes; the previous active generation was preserved"; + try { + registryService.fail(operation.getId(), diagnostic); + log.warn("Failed abandoned RAG generation operation {} for branch {}", + operation.getId(), operation.getBranchName()); + } catch (Exception failure) { + log.error("Could not terminalize abandoned RAG generation operation {}", + operation.getId(), failure); + } + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java new file mode 100644 index 000000000..2aa28b55b --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java @@ -0,0 +1,77 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; + +/** Removes only expired PR-target generations explicitly classified transient. */ +@Service +public class RagTransientBranchIndexCleanupService { + private static final Logger log = LoggerFactory.getLogger( + RagTransientBranchIndexCleanupService.class); + + private final RagBranchIndexRepository branchRepository; + private final RagBranchIndexGenerationRepository generationRepository; + private final RagPipelineClient pipelineClient; + + public RagTransientBranchIndexCleanupService( + RagBranchIndexRepository branchRepository, + RagBranchIndexGenerationRepository generationRepository, + RagPipelineClient pipelineClient) { + this.branchRepository = branchRepository; + this.generationRepository = generationRepository; + this.pipelineClient = pipelineClient; + } + + @Scheduled( + fixedDelayString = "${codecrow.rag.transient.cleanup-interval-ms:3600000}", + initialDelayString = "${codecrow.rag.transient.cleanup-initial-delay-ms:300000}") + @Transactional + public void cleanupExpired() { + OffsetDateTime now = OffsetDateTime.now(); + for (RagBranchIndex index : branchRepository.findByIndexKind( + RagBranchIndexKind.TRANSIENT)) { + var project = index.getProject(); + var config = project.getConfiguration() != null + ? project.getConfiguration().ragConfig() + : null; + int retentionDays = config != null + ? config.getEffectiveBranchRetentionDays() + : 90; + OffsetDateTime lastUse = index.getLastAccessedAt() != null + ? index.getLastAccessedAt() + : index.getUpdatedAt(); + if (lastUse == null || !lastUse.isBefore(now.minusDays(retentionDays))) { + continue; + } + + boolean removed = true; + for (var generation : generationRepository + .findByBranchIndexIdOrderByCreatedAtDesc(index.getId())) { + try { + removed &= pipelineClient.deleteBranch( + project.getWorkspace().getName(), project.getNamespace(), + index.getBranchName(), generation.getCollectionName()); + } catch (Exception failure) { + removed = false; + log.warn("Failed to clean transient RAG generation {}: {}", + generation.getId(), failure.getMessage()); + } + } + if (removed) { + branchRepository.delete(index); + log.info("Removed expired transient RAG branch index project={}, branch={}", + project.getId(), index.getBranchName()); + } + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java index 683983960..c6c0473d4 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java @@ -4,13 +4,17 @@ import okhttp3.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.IOException; +import java.nio.file.Path; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; @Service public class RagPipelineClient { @@ -70,6 +74,42 @@ public Map indexRepository( String commit, List includePatterns, List excludePatterns + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, null); + } + + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, collectionTarget, false, false); + } + + /** + * Index an immutable generation and optionally publish its readable branch + * and legacy-project aliases in the same Qdrant transaction. + */ + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias ) throws IOException { if (!ragEnabled) { log.debug("RAG indexing disabled, skipping repository indexing"); @@ -82,6 +122,19 @@ public Map indexRepository( payload.put("project", projectNamespace); payload.put("branch", branch); payload.put("commit", commit); + if (collectionTarget != null && !collectionTarget.isBlank()) { + payload.put("collection_target", collectionTarget); + } + if (publishBranchAlias) { + payload.put("publish_branch_alias", true); + } + if (publishLegacyProjectAlias) { + payload.put("publish_legacy_project_alias", true); + } + payload.put( + "source_tree_sha256", + RepositorySourceTreeIdentity.sha256(Path.of(repoPath)) + ); if (includePatterns != null && !includePatterns.isEmpty()) { payload.put("include_patterns", includePatterns); } @@ -93,6 +146,74 @@ public Map indexRepository( return postLongRunning(url, payload); } + /** + * Build an index through the progress-streaming transport. The regular + * JSON method above remains available for legacy callers; this overload is + * used by explicit branch maintenance so detailed batch events can reach + * the operator without affecting index correctness. + */ + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + Consumer> progressConsumer + ) throws IOException { + return indexRepository( + repoPath, projectWorkspace, projectNamespace, branch, commit, + includePatterns, excludePatterns, collectionTarget, false, false, + progressConsumer); + } + + /** Streaming variant of exact generation indexing with alias publication. */ + public Map indexRepository( + String repoPath, + String projectWorkspace, + String projectNamespace, + String branch, + String commit, + List includePatterns, + List excludePatterns, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias, + Consumer> progressConsumer + ) throws IOException { + if (!ragEnabled) { + log.debug("RAG indexing disabled, skipping repository indexing"); + return Map.of("status", "skipped", "reason", "RAG disabled"); + } + + Map payload = new HashMap<>(); + payload.put("repo_path", repoPath); + payload.put("workspace", projectWorkspace); + payload.put("project", projectNamespace); + payload.put("branch", branch); + payload.put("commit", commit); + if (collectionTarget != null && !collectionTarget.isBlank()) { + payload.put("collection_target", collectionTarget); + } + if (publishBranchAlias) { + payload.put("publish_branch_alias", true); + } + if (publishLegacyProjectAlias) { + payload.put("publish_legacy_project_alias", true); + } + payload.put("source_tree_sha256", RepositorySourceTreeIdentity.sha256(Path.of(repoPath))); + if (includePatterns != null && !includePatterns.isEmpty()) { + payload.put("include_patterns", includePatterns); + } + if (excludePatterns != null && !excludePatterns.isEmpty()) { + payload.put("exclude_patterns", excludePatterns); + } + return postLongRunningSse( + ragApiUrl + "/index/repository/stream", payload, progressConsumer); + } + public Map updateFiles( List filePaths, String repoBase, @@ -179,6 +300,98 @@ public Map applyChanges( return postLongRunning(ragApiUrl + "/index/apply-changes", payload); } + public Map advanceGeneration( + List updatedFilePaths, + List deletedFilePaths, + String repoBase, + String workspace, + String project, + String branch, + String sourceCommit, + String commit, + String sourceTreeSha256, + String sourceCollectionTarget, + String collectionTarget + ) throws IOException { + return advanceGeneration( + updatedFilePaths, deletedFilePaths, repoBase, workspace, project, + branch, sourceCommit, commit, sourceTreeSha256, + sourceCollectionTarget, collectionTarget, false, false); + } + + /** Advance an exact generation and atomically update its readable aliases. */ + public Map advanceGeneration( + List updatedFilePaths, + List deletedFilePaths, + String repoBase, + String workspace, + String project, + String branch, + String sourceCommit, + String commit, + String sourceTreeSha256, + String sourceCollectionTarget, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias + ) throws IOException { + if (!ragEnabled) { + log.debug("RAG indexing disabled, skipping generation advance"); + return Map.of("status", "skipped", "reason", "RAG disabled"); + } + + Map payload = new HashMap<>(); + payload.put("updated_file_paths", updatedFilePaths); + payload.put("deleted_file_paths", deletedFilePaths); + if (repoBase != null && !repoBase.isBlank()) { + payload.put("repo_base", repoBase); + } + payload.put("workspace", workspace); + payload.put("project", project); + payload.put("branch", branch); + payload.put("source_commit", sourceCommit); + payload.put("commit", commit); + payload.put("source_tree_sha256", sourceTreeSha256); + payload.put("source_collection_target", sourceCollectionTarget); + payload.put("collection_target", collectionTarget); + if (publishBranchAlias) { + payload.put("publish_branch_alias", true); + } + if (publishLegacyProjectAlias) { + payload.put("publish_legacy_project_alias", true); + } + + return postLongRunning(ragApiUrl + "/index/advance-generation", payload); + } + + /** + * Idempotently repair human-readable aliases of one completed generation. + * Exact analysis never depends on this convenience mapping. + */ + public void publishGenerationAliases( + String workspace, + String project, + String branch, + String commit, + String collectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias) throws IOException { + if (!ragEnabled || !publishBranchAlias) { + return; + } + Map payload = new HashMap<>(); + payload.put("workspace", workspace); + payload.put("project", project); + payload.put("branch", branch); + payload.put("commit", commit); + payload.put("collection_target", collectionTarget); + payload.put("publish_branch_alias", true); + if (publishLegacyProjectAlias) { + payload.put("publish_legacy_project_alias", true); + } + post(ragApiUrl + "/index/generation-aliases", payload); + } + public Map getPRContext( String workspace, String project, @@ -341,16 +554,29 @@ public boolean deletePrFiles(String workspace, String project, int prNumber) { * Python endpoint: DELETE /index/{workspace}/{project}/branch/{branch} */ public boolean deleteBranch(String workspace, String project, String branch) throws IOException { + return deleteBranch(workspace, project, branch, null); + } + + public boolean deleteBranch( + String workspace, + String project, + String branch, + String collectionTarget + ) throws IOException { if (!ragEnabled) { return false; } // URL-encode branch name to handle slashes (e.g., feature/xyz -> feature%2Fxyz) String encodedBranch = java.net.URLEncoder.encode(branch, java.nio.charset.StandardCharsets.UTF_8); - String url = String.format("%s/index/%s/%s/branch/%s", ragApiUrl, workspace, project, encodedBranch); + HttpUrl.Builder urlBuilder = HttpUrl.get(String.format( + "%s/index/%s/%s/branch/%s", ragApiUrl, workspace, project, encodedBranch)).newBuilder(); + if (collectionTarget != null && !collectionTarget.isBlank()) { + urlBuilder.addQueryParameter("collection_target", collectionTarget); + } Request.Builder builder = new Request.Builder() - .url(url) + .url(urlBuilder.build()) .delete(); addAuthHeader(builder); Request request = builder.build(); @@ -531,6 +757,59 @@ private Map postLongRunning(String url, Map payl return doRequest(url, payload, longRunningHttpClient); } + @SuppressWarnings("unchecked") + private Map postLongRunningSse( + String url, + Map payload, + Consumer> progressConsumer + ) throws IOException { + RequestBody body = RequestBody.create(objectMapper.writeValueAsString(payload), JSON); + Request.Builder builder = new Request.Builder() + .url(url) + .header("Accept", "text/event-stream") + .post(body); + addAuthHeader(builder); + + try (Response response = longRunningHttpClient.newCall(builder.build()).execute()) { + if (!response.isSuccessful()) { + String detail = response.body() != null ? response.body().string() : "{}"; + throw new IOException("RAG API error: " + response.code() + " — " + detail); + } + if (response.body() == null) { + throw new IOException("RAG progress stream returned no body"); + } + String line; + while ((line = response.body().source().readUtf8Line()) != null) { + if (!line.startsWith("data:")) { + continue; + } + String json = line.substring(5).trim(); + if (json.isEmpty()) { + continue; + } + Map event = objectMapper.readValue(json, Map.class); + String type = String.valueOf(event.get("type")); + if ("progress".equals(type)) { + if (progressConsumer != null) { + progressConsumer.accept(new LinkedHashMap<>(event)); + } + continue; + } + if ("complete".equals(type)) { + Object result = event.get("result"); + if (result instanceof Map resultMap) { + return new LinkedHashMap<>((Map) resultMap); + } + throw new IOException("RAG progress stream completed without index result"); + } + if ("error".equals(type)) { + throw new IOException("RAG API error: " + event.getOrDefault("message", "unknown error")); + } + } + } + throw new IOException("RAG progress stream ended without a terminal result"); + } + /** * Adds the x-service-secret header to the request if a secret is configured. */ diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java index 631134a62..a51873b5d 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java @@ -7,6 +7,7 @@ import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.slf4j.Logger; @@ -95,6 +96,51 @@ public Map performIncrementalUpdate( Set addedFiles, Set modifiedFiles, Set deletedFiles) throws IOException { + return performIncrementalUpdate( + project, vcsConnection, workspaceSlug, repoSlug, branch, + commitHash, addedFiles, modifiedFiles, deletedFiles, + null, null, null, false, false); + } + + public Map performIncrementalUpdate( + Project project, + VcsConnection vcsConnection, + String workspaceSlug, + String repoSlug, + String branch, + String commitHash, + Set addedFiles, + Set modifiedFiles, + Set deletedFiles, + String sourceRevision, + String sourceCollectionTarget, + String targetCollectionTarget) throws IOException { + return performIncrementalUpdate( + project, vcsConnection, workspaceSlug, repoSlug, branch, + commitHash, addedFiles, modifiedFiles, deletedFiles, + sourceRevision, sourceCollectionTarget, targetCollectionTarget, + false, false); + } + + /** + * Advances an immutable generation and, for retained branches, publishes + * operator aliases in the same RAG-side activation transaction. + */ + public Map performIncrementalUpdate( + Project project, + VcsConnection vcsConnection, + String workspaceSlug, + String repoSlug, + String branch, + String commitHash, + Set addedFiles, + Set modifiedFiles, + Set deletedFiles, + String sourceRevision, + String sourceCollectionTarget, + String targetCollectionTarget, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias) throws IOException { Set skippedNonTextFiles = new LinkedHashSet<>(); Set indexableAddedFiles = filterTextCandidates(addedFiles, skippedNonTextFiles); Set indexableModifiedFiles = filterTextCandidates(modifiedFiles, skippedNonTextFiles); @@ -119,7 +165,11 @@ public Map performIncrementalUpdate( List orderedAddedOrModifiedFiles = new ArrayList<>(sortedList(addedOrModifiedFiles)); List orderedDeletedFiles = sortedList(deletedFiles); - if (orderedAddedOrModifiedFiles.isEmpty() && orderedDeletedFiles.isEmpty()) { + boolean generationAdvanceRequested = sourceRevision != null + && sourceCollectionTarget != null + && targetCollectionTarget != null; + if (orderedAddedOrModifiedFiles.isEmpty() && orderedDeletedFiles.isEmpty() + && !generationAdvanceRequested) { result.put("status", "completed"); return result; } @@ -128,31 +178,37 @@ public Map performIncrementalUpdate( try { String revision = commitHash != null && !commitHash.isBlank() ? commitHash : branch; String repoBase = null; - if (!orderedAddedOrModifiedFiles.isEmpty()) { + String sourceTreeSha256 = null; + if (!orderedAddedOrModifiedFiles.isEmpty() || generationAdvanceRequested) { tempDir = Files.createTempDirectory("codecrow-rag-incremental-", PosixFilePermissions.asFileAttribute( PosixFilePermissions.fromString("rwxrwxrwx"))); int effectiveArchiveFileThreshold = fileRetrievalPolicy.archiveFileThreshold(); - boolean useArchive = - fileRetrievalPolicy.shouldUseArchive(orderedAddedOrModifiedFiles.size()); + boolean useArchive = generationAdvanceRequested + || fileRetrievalPolicy.shouldUseArchive(orderedAddedOrModifiedFiles.size()); Set fetchedFilePaths; Set presentFilePaths = Collections.emptySet(); String fileFetchMode; if (useArchive) { log.info("Using one repository archive at revision {} for {} incremental RAG files " - + "(threshold: {})", - revision, orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold); + + "(threshold: {}, exact generation: {})", + revision, orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold, + generationAdvanceRequested); BranchArchiveService.ArchiveDirectorySnapshot archiveSnapshot = branchArchiveService.downloadAndExtractSnapshotToDirectory( vcsConnection, workspaceSlug, repoSlug, revision, - new LinkedHashSet<>(orderedAddedOrModifiedFiles), + generationAdvanceRequested + ? null + : new LinkedHashSet<>(orderedAddedOrModifiedFiles), tempDir); fetchedFilePaths = archiveSnapshot.extractedFiles(); presentFilePaths = archiveSnapshot.presentFiles(); - fileFetchMode = "archive"; + fileFetchMode = generationAdvanceRequested + ? "exact-generation-archive" + : "archive"; } else { log.info("Using per-file VCS retrieval for {} incremental RAG files (threshold: {})", orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold); @@ -209,6 +265,10 @@ public Map performIncrementalUpdate( + revision + ": " + String.join(", ", missingFiles)); } repoBase = tempDir.toString(); + if (generationAdvanceRequested) { + sourceTreeSha256 = RepositorySourceTreeIdentity.sha256(tempDir); + result.put("sourceTreeSha256", sourceTreeSha256); + } result.put("updatedFiles", orderedAddedOrModifiedFiles.size()); int appliedAddedFiles = (int) orderedAddedOrModifiedFiles.stream() .filter(indexableAddedFiles::contains) @@ -219,17 +279,36 @@ public Map performIncrementalUpdate( } String changeSetRepoBase = repoBase; + String targetSourceTreeSha256 = sourceTreeSha256; List filesToUpdate = List.copyOf(orderedAddedOrModifiedFiles); + boolean generationAdvance = generationAdvanceRequested; Map updateResult = executeWithRetry( - "apply incremental RAG change set", - () -> ragPipelineClient.applyChanges( - filesToUpdate, - orderedDeletedFiles, - changeSetRepoBase, - projectWorkspace, - projectNamespace, - branch, - revision)); + generationAdvance + ? "advance immutable RAG generation" + : "apply incremental RAG change set", + () -> generationAdvance + ? ragPipelineClient.advanceGeneration( + filesToUpdate, + orderedDeletedFiles, + changeSetRepoBase, + projectWorkspace, + projectNamespace, + branch, + sourceRevision, + revision, + targetSourceTreeSha256, + sourceCollectionTarget, + targetCollectionTarget, + publishBranchAlias, + publishLegacyProjectAlias) + : ragPipelineClient.applyChanges( + filesToUpdate, + orderedDeletedFiles, + changeSetRepoBase, + projectWorkspace, + projectNamespace, + branch, + revision)); result.put("deletedFiles", orderedDeletedFiles.size()); result.putAll(updateResult); log.info( diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java new file mode 100644 index 000000000..d6c4a9268 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java @@ -0,0 +1,264 @@ +package org.rostilos.codecrow.ragengine.service; + +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.*; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.OffsetDateTime; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; + +/** + * Owns durable branch-index identities and their immutable generations. It does + * not know how Qdrant or a VCS works; callers build and validate the physical + * generation, then atomically publish or fail it through this service. + */ +@Service +public class RagBranchIndexRegistryService { + + private final RagBranchIndexRepository branchIndexRepository; + private final RagBranchIndexGenerationRepository generationRepository; + private final RagIndexOperationRepository operationRepository; + + public RagBranchIndexRegistryService( + RagBranchIndexRepository branchIndexRepository, + RagBranchIndexGenerationRepository generationRepository, + RagIndexOperationRepository operationRepository) { + this.branchIndexRepository = branchIndexRepository; + this.generationRepository = generationRepository; + this.operationRepository = operationRepository; + } + + public record BuildRegistration( + RagBranchIndex branchIndex, + RagBranchIndexGeneration generation, + RagIndexOperation operation, + boolean existingOperation) { + } + + @Transactional + public BuildRegistration registerBuild( + Project project, + String branchName, + RagBranchIndexKind indexKind, + String fromRevision, + String toRevision, + String representationFingerprint) { + requireProjectIdentity(project); + String branch = requireText(branchName, "branchName"); + String targetRevision = requireText(toRevision, "toRevision"); + RagBranchIndexKind kind = indexKind != null ? indexKind : RagBranchIndexKind.DURABLE; + + String operationKey = operationKey( + project.getId(), branch, fromRevision, targetRevision, representationFingerprint); + Optional existing = operationRepository + .findByProjectIdAndOperationKey(project.getId(), operationKey); + if (existing.isPresent()) { + RagIndexOperation operation = existing.get(); + return new BuildRegistration( + operation.getGeneration().getBranchIndex(), + operation.getGeneration(), + operation, + true); + } + + RagBranchIndex branchIndex = branchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branch) + .orElseGet(() -> new RagBranchIndex(project, branch, kind)); + if (branchIndex.getIndexKind() == RagBranchIndexKind.LEGACY + || kind == RagBranchIndexKind.PRIMARY + || (branchIndex.getIndexKind() == RagBranchIndexKind.TRANSIENT + && kind == RagBranchIndexKind.DURABLE)) { + branchIndex.setIndexKind(kind); + } + branchIndex.requestRevision(targetRevision); + branchIndex = branchIndexRepository.save(branchIndex); + + RagBranchIndexGeneration parent = branchIndex.getActiveGeneration(); + String collectionName = physicalCollectionName(project, branch, targetRevision, operationKey); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, + targetRevision, + collectionName, + parent, + fromRevision, + representationFingerprint); + generation = generationRepository.save(generation); + + RagIndexOperation operation = new RagIndexOperation( + project, branch, fromRevision, targetRevision, operationKey); + operation.setGeneration(generation); + operation = operationRepository.save(operation); + + return new BuildRegistration(branchIndex, generation, operation, false); + } + + @Transactional + public void startBuild(long operationId, Long jobId) { + RagIndexOperation operation = requireOperation(operationId); + if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { + return; + } + if (operation.getStatus() == RagIndexOperationStatus.FAILED) { + operation.getGeneration().retry(); + generationRepository.save(operation.getGeneration()); + operation.getGeneration().getBranchIndex().requestRevision( + operation.getToRevision()); + branchIndexRepository.save( + operation.getGeneration().getBranchIndex()); + } + operation.setJobId(jobId); + operation.start(); + operationRepository.save(operation); + } + + @Transactional + public RagBranchIndexGeneration publish( + long operationId, + String manifestDigest, + int fileCount, + int chunkCount) { + RagIndexOperation operation = requireOperation(operationId); + RagBranchIndexGeneration generation = operation.getGeneration(); + if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { + return generation; + } + if (generation.getStatus() != RagBranchIndexGenerationStatus.BUILDING) { + throw new IllegalStateException("Only a building generation can be published"); + } + + RagBranchIndex branchIndex = generation.getBranchIndex(); + RagBranchIndexGeneration previous = branchIndex.getActiveGeneration(); + generation.activate(requireText(manifestDigest, "manifestDigest"), fileCount, chunkCount); + generationRepository.save(generation); + if (previous != null && previous.getId() != null && !previous.getId().equals(generation.getId())) { + previous.supersede(); + generationRepository.save(previous); + } + branchIndex.activate(generation); + branchIndexRepository.save(branchIndex); + operation.succeed(generation); + operationRepository.save(operation); + return generation; + } + + @Transactional + public void fail(long operationId, String errorMessage) { + RagIndexOperation operation = requireOperation(operationId); + if (operation.getStatus() == RagIndexOperationStatus.SUCCEEDED) { + return; + } + String failure = requireText(errorMessage, "errorMessage"); + operation.getGeneration().fail(failure); + generationRepository.save(operation.getGeneration()); + operation.getGeneration().getBranchIndex().failUpdate(failure); + branchIndexRepository.save(operation.getGeneration().getBranchIndex()); + operation.fail(failure); + operationRepository.save(operation); + } + + @Transactional + public Optional findAvailableGeneration( + long projectId, + String branchName, + String revision) { + Optional branchIndex = branchIndexRepository + .findByProjectIdAndBranchName(projectId, requireText(branchName, "branchName")); + if (branchIndex.isEmpty()) { + return Optional.empty(); + } + RagBranchIndex index = branchIndex.get(); + index.markAccessed(); + branchIndexRepository.save(index); + if (index.getActiveGeneration() != null + && index.getActiveGeneration().getRevision().equals(revision)) { + return Optional.of(index.getActiveGeneration()); + } + return generationRepository + .findFirstByBranchIndexIdAndRevisionAndStatusInOrderByCreatedAtDesc( + index.getId(), revision, + List.of(RagBranchIndexGenerationStatus.ACTIVE, + RagBranchIndexGenerationStatus.SUPERSEDED)); + } + + @Transactional + public void heartbeatBuild(long operationId) { + RagIndexOperation operation = requireOperation(operationId); + operation.heartbeat(); + operationRepository.save(operation); + } + + public List findRecoverableOperations(OffsetDateTime updatedBefore) { + return operationRepository.findByStatusInAndUpdatedAtBefore( + List.of(RagIndexOperationStatus.PENDING, RagIndexOperationStatus.RUNNING), + updatedBefore); + } + + static String physicalCollectionName( + Project project, + String branchName, + String revision, + String operationKey) { + long workspaceId = project.getWorkspace() != null && project.getWorkspace().getId() != null + ? project.getWorkspace().getId() + : 0L; + return "cc_w" + workspaceId + + "_p" + project.getId() + + "_b" + digest(branchName).substring(0, 12) + + "_r" + digest(revision).substring(0, 12) + + "_g" + operationKey.substring(0, 12); + } + + static String operationKey( + long projectId, + String branchName, + String fromRevision, + String toRevision, + String representationFingerprint) { + return digest(projectId + "\n" + + branchName + "\n" + + nullToEmpty(fromRevision) + "\n" + + toRevision + "\n" + + nullToEmpty(representationFingerprint)); + } + + private RagIndexOperation requireOperation(long operationId) { + return operationRepository.findById(operationId) + .orElseThrow(() -> new IllegalArgumentException("RAG index operation not found: " + operationId)); + } + + private static void requireProjectIdentity(Project project) { + if (project == null || project.getId() == null) { + throw new IllegalArgumentException("A persisted project is required for branch indexing"); + } + } + + private static String requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " is required"); + } + return value.trim(); + } + + private static String nullToEmpty(String value) { + return value == null ? "" : value; + } + + private static String digest(String value) { + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java index dbca6ad2c..bf71b9862 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java @@ -7,18 +7,23 @@ import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.rostilos.codecrow.core.service.AnalysisJobService; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import java.io.IOException; @@ -32,11 +37,10 @@ import java.util.function.Consumer; /** - * Implementation of RagOperationsService using single-collection-per-project - * architecture. - * - * Each project has ONE Qdrant collection containing all branches. - * Branch is stored as metadata in each point, allowing multi-branch queries. + * Coordinates both legacy project collections and exact branch generations. + * Existing projects retain the shared collection path until exact branch + * indexing is configured; configured branches use immutable, branch-bound + * generation targets selected through the registry. */ @Service public class RagOperationsServiceImpl implements RagOperationsService { @@ -50,6 +54,11 @@ public class RagOperationsServiceImpl implements RagOperationsService { private final RagBranchIndexRepository ragBranchIndexRepository; private final VcsClientProvider vcsClientProvider; private final RagPipelineClient ragPipelineClient; + private final RagBranchIndexRegistryService branchIndexRegistryService; + private final BranchIndexGenerationBuildService branchGenerationBuildService; + + @Autowired(required = false) + private RagBranchIndexGenerationRepository branchGenerationRepository; @Value("${codecrow.rag.api.enabled:true}") private boolean ragApiEnabled; @@ -62,6 +71,38 @@ public RagOperationsServiceImpl( RagBranchIndexRepository ragBranchIndexRepository, VcsClientProvider vcsClientProvider, RagPipelineClient ragPipelineClient) { + this(ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, null, null); + } + + public RagOperationsServiceImpl( + RagIndexTrackingService ragIndexTrackingService, + IncrementalRagUpdateService incrementalRagUpdateService, + AnalysisLockService analysisLockService, + AnalysisJobService analysisJobService, + RagBranchIndexRepository ragBranchIndexRepository, + VcsClientProvider vcsClientProvider, + RagPipelineClient ragPipelineClient, + RagBranchIndexRegistryService branchIndexRegistryService) { + this(ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, branchIndexRegistryService, null); + } + + @Autowired + public RagOperationsServiceImpl( + RagIndexTrackingService ragIndexTrackingService, + IncrementalRagUpdateService incrementalRagUpdateService, + AnalysisLockService analysisLockService, + AnalysisJobService analysisJobService, + RagBranchIndexRepository ragBranchIndexRepository, + VcsClientProvider vcsClientProvider, + RagPipelineClient ragPipelineClient, + RagBranchIndexRegistryService branchIndexRegistryService, + BranchIndexGenerationBuildService branchGenerationBuildService) { this.ragIndexTrackingService = ragIndexTrackingService; this.incrementalRagUpdateService = incrementalRagUpdateService; this.analysisLockService = analysisLockService; @@ -69,6 +110,8 @@ public RagOperationsServiceImpl( this.ragBranchIndexRepository = ragBranchIndexRepository; this.vcsClientProvider = vcsClientProvider; this.ragPipelineClient = ragPipelineClient; + this.branchIndexRegistryService = branchIndexRegistryService; + this.branchGenerationBuildService = branchGenerationBuildService; } @Override @@ -132,8 +175,26 @@ public boolean triggerIncrementalUpdate( return false; } - String effectiveRawDiff = resolveDiffFromCompletedCheckpoint( - project, branchName, commitHash, rawDiff); + boolean exactGenerationMode = usesExactGenerations(project); + RagBranchIndexKind exactGenerationKind = exactGenerationMode + ? indexKind(project, branchName) : null; + boolean publishBranchAlias = exactGenerationKind == RagBranchIndexKind.PRIMARY + || exactGenerationKind == RagBranchIndexKind.DURABLE; + boolean publishLegacyProjectAlias = exactGenerationKind + == RagBranchIndexKind.PRIMARY; + RagBranchIndexGeneration initialSourceGeneration = exactGenerationMode + ? ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branchName) + .map(RagBranchIndex::getActiveGeneration) + .orElse(null) + : null; + boolean fullExactSnapshotRequired = exactGenerationMode + && initialSourceGeneration == null; + + String effectiveRawDiff = fullExactSnapshotRequired + ? "" + : resolveDiffFromCompletedCheckpoint( + project, branchName, commitHash, rawDiff); log.info("RAG checkpoint reconciliation complete; parsing effective diff..."); // Parse the diff to find changed files @@ -147,7 +208,8 @@ public boolean triggerIncrementalUpdate( log.info("Diff parsed: added={}, modified={}, deleted={}", addedFiles, modifiedFiles, deletedFiles); - if (addedOrModifiedSize == 0 && deletedFiles.isEmpty()) { + if (addedOrModifiedSize == 0 && deletedFiles.isEmpty() + && !exactGenerationMode) { log.info("Skipping RAG incremental update - no files changed in diff"); return true; } @@ -198,17 +260,90 @@ public boolean triggerIncrementalUpdate( String workspaceSlug = vcsRepoBinding.getExternalNamespace(); String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - // Perform the actual incremental update - Map result = incrementalRagUpdateService.performIncrementalUpdate( - project, - vcsConnection, - workspaceSlug, - repoSlug, - branchName, - commitHash, - addedFiles, - modifiedFiles, - deletedFiles); + RagBranchIndexGeneration sourceGeneration = null; + RagBranchIndexRegistryService.BuildRegistration branchBuild = null; + if (exactGenerationMode) { + sourceGeneration = initialSourceGeneration; + if (sourceGeneration != null) { + branchBuild = branchIndexRegistryService.registerBuild( + project, + branchName, + exactGenerationKind, + sourceGeneration.getRevision(), + commitHash, + sourceGeneration.getRepresentationFingerprint()); + branchIndexRegistryService.startBuild( + branchBuild.operation().getId(), + job != null ? job.getId() : null); + } + } + + Map result; + try { + if (fullExactSnapshotRequired) { + var ragConfig = project.getConfiguration().ragConfig(); + result = branchGenerationBuildService.build( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + exactGenerationKind, + ragConfig.includePatterns(), + ragConfig.excludePatterns(), + job != null ? job.getId() : null); + } else if (exactGenerationMode) { + result = incrementalRagUpdateService.performIncrementalUpdate( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + addedFiles, + modifiedFiles, + deletedFiles, + sourceGeneration.getRevision(), + sourceGeneration.getCollectionName(), + branchBuild.generation().getCollectionName(), + publishBranchAlias, + publishLegacyProjectAlias); + } else { + result = incrementalRagUpdateService.performIncrementalUpdate( + project, + vcsConnection, + workspaceSlug, + repoSlug, + branchName, + commitHash, + addedFiles, + modifiedFiles, + deletedFiles); + } + if (exactGenerationMode && !fullExactSnapshotRequired) { + Object digest = result.get("generation_manifest_sha256"); + if (!(digest instanceof String manifestDigest) + || manifestDigest.isBlank()) { + throw new IllegalStateException( + "Advanced RAG generation has no manifest digest"); + } + branchIndexRegistryService.publish( + branchBuild.operation().getId(), + manifestDigest, + ((Number) result.getOrDefault("document_count", 0)).intValue(), + ((Number) result.getOrDefault("chunk_count", 0)).intValue()); + } + } catch (Exception generationFailure) { + if (branchBuild != null) { + branchIndexRegistryService.fail( + branchBuild.operation().getId(), + generationFailure.getMessage() != null + ? generationFailure.getMessage() + : generationFailure.getClass().getSimpleName()); + } + throw generationFailure; + } int filesUpdated = (Integer) result.getOrDefault("updatedFiles", 0); int filesDeleted = (Integer) result.getOrDefault("deletedFiles", 0); @@ -411,9 +546,6 @@ public boolean updateBranchIndex( Project project, String targetBranch, Consumer> eventConsumer) { - // Update branch index - calculates diff between base branch and target branch - // Unlike ensureBranchIndexForPrTarget, this always recalculates the full diff - if (!isRagEnabled(project)) { log.debug("RAG not enabled for project={}", project.getId()); return false; @@ -424,6 +556,17 @@ public boolean updateBranchIndex( return false; } + if (!targetBranch.equals(getBaseBranch(project)) + && !shouldHaveBranchIndex(project, targetBranch)) { + log.info("Skipping branch index update for non-retained branch: project={}, branch={}", + project.getId(), targetBranch); + eventConsumer.accept(Map.of( + "type", "info", + "state", "rag_skipped", + "message", "Branch is not configured as a retained RAG branch")); + return false; + } + // Get VCS connection info VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); if (vcsRepoBinding == null) { @@ -445,7 +588,28 @@ public boolean updateBranchIndex( try { VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - log.info("Updating branch index for project={}, branch={} (diff vs {})", + String targetCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); + Optional completedBranchIndex = ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), targetBranch) + .filter(index -> index.getCommitHash() != null && !index.getCommitHash().isBlank()); + + if (completedBranchIndex.isPresent()) { + String checkpoint = completedBranchIndex.get().getCommitHash(); + if (checkpoint.equals(targetCommit)) { + log.info("Branch index already represents project={}, branch={}, commit={}", + project.getId(), targetBranch, targetCommit); + return true; + } + + log.info("Updating branch index from completed checkpoint: project={}, branch={}, {}..{}", + project.getId(), targetBranch, checkpoint, targetCommit); + // triggerIncrementalUpdate resolves the exact checkpoint range. Passing an + // empty supplied diff avoids the former redundant base-to-target compare. + return triggerIncrementalUpdate( + project, targetBranch, targetCommit, "", eventConsumer); + } + + log.info("Seeding legacy branch index for project={}, branch={} (diff vs {})", project.getId(), targetBranch, baseBranch); eventConsumer.accept(Map.of( @@ -453,7 +617,8 @@ public boolean updateBranchIndex( "state", "branch_index", "message", String.format("Calculating diff between '%s' and '%s'", baseBranch, targetBranch))); - // Always get fresh diff between base branch and target branch + // Compatibility path for a branch that has no checkpoint yet. The exact + // generation builder replaces this with a complete verified snapshot. String rawDiff = vcsClient.getBranchDiff(workspaceSlug, repoSlug, baseBranch, targetBranch); if (rawDiff == null || rawDiff.isEmpty()) { @@ -464,8 +629,6 @@ public boolean updateBranchIndex( return true; } - String targetCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); - log.info("Branch diff found: {} bytes, triggering incremental update for branch={}, commit={}", rawDiff.length(), targetBranch, targetCommit); @@ -519,7 +682,7 @@ public boolean ensureBranchIndexForPrTarget( String repoSlug = vcsRepoBinding.getExternalRepoSlug(); // Get base branch (main branch) - String baseBranch = getBaseBranch(project); + String baseBranch = getBaseBranch(project); // Same branch? Already indexed via main index if (targetBranch.equals(baseBranch)) { @@ -527,6 +690,27 @@ public boolean ensureBranchIndexForPrTarget( return true; } + // Exact-generation mode seeds an immutable snapshot of the target + // revision. It must not first compute the potentially enormous + // primary-to-target diff that motivated multi-branch indexing. + if (usesExactGenerations(project)) { + if (!shouldHaveBranchIndex(project, targetBranch) + && !shouldCreateTransientBranchIndex(project, targetBranch)) { + return false; + } + try { + VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); + String targetCommit = vcsClient.getLatestCommitHash( + workspaceSlug, repoSlug, targetBranch); + return triggerIncrementalUpdate( + project, targetBranch, targetCommit, "", eventConsumer); + } catch (Exception failure) { + log.warn("Failed to seed exact branch generation for project={}, branch={}: {}", + project.getId(), targetBranch, failure.getMessage()); + return false; + } + } + // Check if branch already has indexed data (RagBranchIndex exists) // Note: We still proceed with diff check to ensure any new changes are indexed boolean branchIndexExists = isBranchIndexReady(project, targetBranch); @@ -616,8 +800,24 @@ public boolean deleteBranchIndex( "state", "branch_delete", "message", String.format("Deleting RAG index for branch '%s'", branchName))); - // Delete from RAG pipeline - boolean success = ragPipelineClient.deleteBranch(workspaceSlug, projectSlug, branchName); + boolean success; + Optional exactIndex = ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branchName) + .filter(index -> index.getActiveGeneration() != null); + if (exactIndex.isPresent() && branchGenerationRepository != null) { + List generations = branchGenerationRepository + .findByBranchIndexIdOrderByCreatedAtDesc(exactIndex.get().getId()); + success = !generations.isEmpty(); + for (RagBranchIndexGeneration generation : generations) { + success &= ragPipelineClient.deleteBranch( + project.getWorkspace().getName(), project.getNamespace(), + branchName, generation.getCollectionName()); + } + } else { + // Backward-compatible cleanup for the legacy shared collection. + success = ragPipelineClient.deleteBranch( + workspaceSlug, projectSlug, branchName); + } if (success) { // Clean up database tracking @@ -769,6 +969,17 @@ public boolean ensureRagIndexUpToDate( eventConsumer); } + if (!shouldHaveBranchIndex(project, targetBranch) + && !shouldCreateTransientBranchIndex(project, targetBranch)) { + log.info("Skipping RAG preparation for non-indexed PR target: project={}, branch={}", + project.getId(), targetBranch); + eventConsumer.accept(Map.of( + "type", "info", + "state", "rag_skipped", + "message", "PR target is not configured for retained or temporary RAG indexing")); + return false; + } + // Case 2: Different branch - ensure main index is ready, then ensure branch is // indexed log.info("Target branch '{}' differs from base branch '{}' - will ensure branch index", targetBranch, @@ -820,6 +1031,17 @@ private boolean ensureMainIndexUpToDate( String indexedCommit = indexStatus.get().getIndexedCommitHash(); + if (usesExactGenerations(project) + && ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branchName) + .map(RagBranchIndex::getActiveGeneration) + .isEmpty()) { + log.info("Creating first exact primary generation for project={}, branch={}, commit={}", + project.getId(), branchName, currentCommit); + return triggerIncrementalUpdate( + project, branchName, currentCommit, "", eventConsumer); + } + // If commits match, index is up to date if (currentCommit.equals(indexedCommit)) { log.debug("Main RAG index is up-to-date for project={}, commit={}", project.getId(), currentCommit); @@ -873,7 +1095,12 @@ private boolean ensureBranchIndexUpToDate( .findByProjectIdAndBranchName(project.getId(), targetBranch); if (branchIndexOpt.isEmpty()) { - // No branch index exists - create it by getting full diff vs main + // The exact path seeds from a complete target snapshot; the legacy + // implementation below retains the former base-diff behavior. + if (usesExactGenerations(project)) { + return triggerIncrementalUpdate( + project, targetBranch, currentCommit, "", eventConsumer); + } log.info("No RagBranchIndex entry found for project={}, branch={} - will create with full diff vs {}", project.getId(), targetBranch, baseBranch); return ensureBranchIndexForPrTarget(project, targetBranch, eventConsumer); @@ -884,6 +1111,11 @@ private boolean ensureBranchIndexUpToDate( log.info("Existing RagBranchIndex for project={}, branch={}: indexedCommit={}", project.getId(), targetBranch, indexedCommit); + if (usesExactGenerations(project) && branchIndex.getActiveGeneration() == null) { + return triggerIncrementalUpdate( + project, targetBranch, currentCommit, "", eventConsumer); + } + // If commits match, index is up to date if (currentCommit.equals(indexedCommit)) { log.info("Branch index is up-to-date for project={}, branch={}, commit={}", @@ -926,4 +1158,22 @@ private boolean ensureBranchIndexUpToDate( project, targetBranch, currentCommit, rawDiff, eventConsumer); } + private boolean usesExactGenerations(Project project) { + return branchIndexRegistryService != null + && branchGenerationBuildService != null + && project != null + && project.getConfiguration() != null + && project.getConfiguration().ragConfig() != null + && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + } + + private RagBranchIndexKind indexKind(Project project, String branchName) { + if (branchName.equals(getBaseBranch(project))) { + return RagBranchIndexKind.PRIMARY; + } + return shouldHaveBranchIndex(project, branchName) + ? RagBranchIndexKind.DURABLE + : RagBranchIndexKind.TRANSIENT; + } + } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java index d43f40f09..92be19167 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java @@ -39,6 +39,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.rostilos.codecrow.queue.RedisQueueService; +import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.scheduling.annotation.Async; @@ -240,6 +241,8 @@ private Map performIndexing( // Extract the downloaded archive locally jobService.logToJob(job, JobLogLevel.INFO, "extraction", "Extracting repository archive..."); extractArchiveFileAndCleanup(tempArchiveFile, tempDir); + String sourceTreeSha256 = + RepositorySourceTreeIdentity.sha256(tempDir); if (includePatterns != null && !includePatterns.isEmpty()) { log.info("Using {} include patterns from project config", includePatterns.size()); @@ -262,6 +265,7 @@ private Map performIndexing( "project", project.getNamespace(), "branch", branch, "commit", commitHash, + "source_tree_sha256", sourceTreeSha256, "preserve_other_branches", config.ragConfig().isMultiBranchEnabled(), "cleanup_repo_path", true, "include_patterns", includePatterns != null ? includePatterns : java.util.List.of(), diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentity.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentity.java new file mode 100644 index 000000000..d6ee89144 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentity.java @@ -0,0 +1,183 @@ +package org.rostilos.codecrow.ragengine.source; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Set; + +/** + * Canonical content identity for a normalized repository source tree. + * + *

The digest intentionally excludes Git's administrative directory. It + * includes every other regular file and symlink using repository-relative + * paths and exact bytes, matching the RAG consumer's verifier.

+ */ +public final class RepositorySourceTreeIdentity { + private static final byte[] SCHEMA = + "codecrow.repository-source-tree".getBytes(StandardCharsets.US_ASCII); + + private RepositorySourceTreeIdentity() { + } + + public static String sha256(Path repositoryRoot) throws IOException { + Path root = repositoryRoot.toAbsolutePath().normalize(); + if (!Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Repository source path is not a directory: " + root); + } + + List entries = new ArrayList<>(); + Files.walkFileTree(root, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory( + Path directory, + BasicFileAttributes attributes + ) { + if (!directory.equals(root) + && ".git".equals(root.relativize(directory).getName(0).toString())) { + return FileVisitResult.SKIP_SUBTREE; + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) + throws IOException { + Path relative = root.relativize(file); + if (relative.getNameCount() > 0 + && ".git".equals(relative.getName(0).toString())) { + return FileVisitResult.CONTINUE; + } + if (attributes.isSymbolicLink()) { + entries.add(new Entry("symlink", relative, Files.readSymbolicLink(file))); + } else if (attributes.isRegularFile()) { + entries.add(new Entry("file", relative, file)); + } else { + throw new IOException( + "Repository source contains an unsupported filesystem entry: " + + relative.toString().replace('\\', '/')); + } + return FileVisitResult.CONTINUE; + } + }); + entries.sort(Comparator.comparing( + entry -> entry.relativePath().toString().replace('\\', '/') + .getBytes(StandardCharsets.UTF_8), + RepositorySourceTreeIdentity::compareUnsigned + )); + + MessageDigest digest = newDigest(); + feedFramed(digest, SCHEMA); + byte[] buffer = new byte[1024 * 1024]; + for (Entry entry : entries) { + feedFramed(digest, entry.kind().getBytes(StandardCharsets.US_ASCII)); + feedFramed( + digest, + entry.relativePath().toString().replace('\\', '/') + .getBytes(StandardCharsets.UTF_8) + ); + if ("symlink".equals(entry.kind())) { + feedFramed( + digest, + entry.value().toString().getBytes(StandardCharsets.UTF_8) + ); + continue; + } + + BasicFileAttributes attributes = Files.readAttributes( + entry.value(), + BasicFileAttributes.class, + LinkOption.NOFOLLOW_LINKS + ); + if (!attributes.isRegularFile()) { + throw new IOException( + "Repository source entry changed or is not a regular file: " + + entry.relativePath().toString().replace('\\', '/')); + } + long expectedSize = attributes.size(); + digest.update(longBytes(expectedSize)); + long observedSize = 0; + Set options = Set.of( + StandardOpenOption.READ, + LinkOption.NOFOLLOW_LINKS + ); + try (SeekableByteChannel channel = Files.newByteChannel( + entry.value(), + options + )) { + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer); + int count; + while ((count = channel.read(byteBuffer)) != -1) { + if (count > 0) { + digest.update(buffer, 0, count); + observedSize += count; + } + byteBuffer.clear(); + } + } + if (observedSize != expectedSize) { + throw new IOException( + "Repository source changed while it was being attested: " + + entry.relativePath().toString().replace('\\', '/')); + } + } + digest.update(longBytes(entries.size())); + return toHex(digest.digest()); + } + + private static int compareUnsigned(byte[] left, byte[] right) { + int length = Math.min(left.length, right.length); + for (int index = 0; index < length; index++) { + int comparison = Integer.compare( + Byte.toUnsignedInt(left[index]), + Byte.toUnsignedInt(right[index]) + ); + if (comparison != 0) { + return comparison; + } + } + return Integer.compare(left.length, right.length); + } + + private static void feedFramed(MessageDigest digest, byte[] value) { + digest.update(longBytes(value.length)); + digest.update(value); + } + + private static byte[] longBytes(long value) { + return ByteBuffer.allocate(Long.BYTES).putLong(value).array(); + } + + private static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + private static String toHex(byte[] bytes) { + StringBuilder result = new StringBuilder(bytes.length * 2); + for (byte value : bytes) { + result.append(Character.forDigit((value >>> 4) & 0xf, 16)); + result.append(Character.forDigit(value & 0xf, 16)); + } + return result.toString(); + } + + private record Entry(String kind, Path relativePath, Path value) { + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java new file mode 100644 index 000000000..92515dd42 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java @@ -0,0 +1,174 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.springframework.test.util.ReflectionTestUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class BranchIndexGenerationBuildServiceTest { + @Mock private BranchArchiveService archiveService; + @Mock private RagPipelineClient pipelineClient; + @Mock private RagBranchIndexRegistryService registryService; + + private BranchIndexGenerationBuildService service; + private Project project; + private RagBranchIndexGeneration generation; + private RagIndexOperation operation; + + @BeforeEach + void setUp() { + service = new BranchIndexGenerationBuildService( + archiveService, pipelineClient, registryService); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + RagBranchIndex branchIndex = new RagBranchIndex( + project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + generation = new RagBranchIndexGeneration( + branchIndex, "develop-400", "opaque-generation-target", + null, null, null); + generation.setId(20L); + operation = new RagIndexOperation( + project, "develop", null, "develop-400", "operation-key"); + operation.setId(30L); + operation.setGeneration(generation); + } + + @Test + void buildsPinnedSnapshotPublishesManifestAndRemovesTemporaryTree() throws Exception { + when(registryService.registerBuild( + project, "develop", RagBranchIndexKind.DURABLE, + null, "develop-400", null)) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + when(pipelineClient.indexRepository( + anyString(), eq("workspace"), eq("namespace"), + eq("develop"), eq("develop-400"), eq(List.of("src/**")), + eq(List.of("vendor/**")), eq("opaque-generation-target"), + eq(true), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400)); + ReflectionTestUtils.setField(project, "namespace", "namespace"); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + + Map result = service.build( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of("src/**"), List.of("vendor/**"), 77L); + + assertThat(result).containsEntry( + "generation_manifest_sha256", "manifest-400"); + verify(registryService).startBuild(30L, 77L); + verify(registryService).publish(30L, "manifest-400", 231, 400); + ArgumentCaptor snapshot = ArgumentCaptor.forClass(Path.class); + verify(archiveService).downloadAndExtractSnapshotToDirectory( + any(), eq("provider-workspace"), eq("repo"), + eq("develop-400"), isNull(), snapshot.capture()); + assertThat(Files.exists(snapshot.getValue())).isFalse(); + } + + @Test + void missingManifestFailsOperationAndDoesNotPublish() throws Exception { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), isNull())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + when(pipelineClient.indexRepository( + anyString(), any(), any(), any(), any(), any(), any(), any(), + anyBoolean(), anyBoolean())) + .thenReturn(Map.of("document_count", 231)); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + assertThatThrownBy(() -> service.build( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of())) + .isInstanceOf(IOException.class) + .hasMessageContaining("no manifest digest"); + + verify(registryService).fail(30L, + "RAG full branch generation has no manifest digest"); + verify(registryService, never()).publish(anyLong(), anyString(), anyInt(), anyInt()); + } + + @Test + void successfulIdempotentOperationIsReusedWithoutProviderOrVectorCalls() throws Exception { + operation.setStatus(RagIndexOperationStatus.SUCCEEDED); + generation.activate("manifest-400", 231, 400); + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), isNull())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, true)); + + Map result = service.build( + project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of()); + + assertThat(result).containsEntry("status", "reused") + .containsEntry("collection_target", "opaque-generation-target") + .containsEntry("generation_manifest_sha256", "manifest-400"); + verifyNoInteractions(archiveService, pipelineClient); + } + + @Test + void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws Exception { + when(registryService.registerBuild( + eq(project), eq("develop"), eq(RagBranchIndexKind.DURABLE), + isNull(), eq("develop-400"), eq("operator-refresh:77"))) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + when(pipelineClient.indexRepository( + anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), + anyList(), anyList(), eq("opaque-generation-target"), + eq(true), eq(false), any())) + .thenReturn(Map.of( + "generation_manifest_sha256", "fresh-manifest", + "document_count", 231, + "chunk_count", 400)); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + service.rebuild(project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of(), 77L, ignored -> { }); + + verify(archiveService).downloadAndExtractSnapshotToDirectory( + any(), eq("provider-workspace"), eq("repo"), eq("develop-400"), isNull(), any()); + verify(registryService).publish(30L, "fresh-manifest", 231, 400); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java new file mode 100644 index 000000000..968961401 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java @@ -0,0 +1,59 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; + +import java.util.List; + +import static org.mockito.Mockito.*; + +class RagBranchOperatorAliasReconciliationServiceTest { + + @Test + void restoresReadableAliasesForDurableAndPrimaryGenerationsOnly() throws Exception { + RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); + RagPipelineClient client = mock(RagPipelineClient.class); + RagBranchOperatorAliasReconciliationService service = + new RagBranchOperatorAliasReconciliationService(repository, client); + + RagBranchIndex primary = index("main", RagBranchIndexKind.PRIMARY, "main-target"); + RagBranchIndex durable = index("develop", RagBranchIndexKind.DURABLE, "develop-target"); + RagBranchIndex transientIndex = index("release", RagBranchIndexKind.TRANSIENT, "release-target"); + when(repository.findAll()).thenReturn(List.of(primary, durable, transientIndex)); + + service.reconcileActiveGenerationAliases(); + + verify(client).publishGenerationAliases( + "workspace", "project", "main", "revision", "main-target", true, true); + verify(client).publishGenerationAliases( + "workspace", "project", "develop", "revision", "develop-target", true, false); + verifyNoMoreInteractions(client); + } + + private static RagBranchIndex index( + String branch, + RagBranchIndexKind kind, + String target) { + Workspace workspace = mock(Workspace.class); + when(workspace.getName()).thenReturn("workspace"); + Project project = mock(Project.class); + when(project.getWorkspace()).thenReturn(workspace); + when(project.getNamespace()).thenReturn("project"); + when(project.getId()).thenReturn(1L); + RagBranchIndexGeneration generation = mock(RagBranchIndexGeneration.class); + when(generation.getRevision()).thenReturn("revision"); + when(generation.getCollectionName()).thenReturn(target); + RagBranchIndex index = mock(RagBranchIndex.class); + when(index.getProject()).thenReturn(project); + when(index.getBranchName()).thenReturn(branch); + when(index.getIndexKind()).thenReturn(kind); + when(index.getActiveGeneration()).thenReturn(generation); + return index; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java new file mode 100644 index 000000000..404706b43 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java @@ -0,0 +1,31 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.rag.RagIndexOperation; +import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; + +import java.time.OffsetDateTime; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +class RagIndexOperationRecoveryServiceTest { + + @Test + void abandonedPersistedOperationBecomesTerminalWithUsefulDiagnostic() { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + RagIndexOperation operation = new RagIndexOperation(); + operation.setId(81L); + operation.setBranchName("develop"); + when(registry.findRecoverableOperations(any(OffsetDateTime.class))) + .thenReturn(List.of(operation)); + + new RagIndexOperationRecoveryService(registry, 30) + .failAbandonedOperations(); + + verify(registry).fail(eq(81L), argThat(message -> + message.contains("stopped heartbeating") + && message.contains("previous active generation was preserved"))); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java new file mode 100644 index 000000000..072fe9969 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java @@ -0,0 +1,59 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.List; + +import static org.mockito.Mockito.*; + +class RagTransientBranchIndexCleanupServiceTest { + + @Test + void deletesExpiredTransientGenerationUsingProjectTenantCoordinates() throws Exception { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagBranchIndexGenerationRepository generations = + mock(RagBranchIndexGenerationRepository.class); + RagPipelineClient pipeline = mock(RagPipelineClient.class); + Project project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + project.setNamespace("namespace"); + Workspace workspace = new Workspace(); + workspace.setName("workspace"); + project.setWorkspace(workspace); + project.setConfiguration(new ProjectConfig( + false, "master", null, + new RagConfig(true, "master", null, null, + true, 30, List.of("develop"), true))); + RagBranchIndex index = new RagBranchIndex( + project, "release/candidate", RagBranchIndexKind.TRANSIENT); + index.setId(10L); + index.setLastAccessedAt(OffsetDateTime.now().minusDays(31)); + RagBranchIndexGeneration generation = mock(RagBranchIndexGeneration.class); + when(generation.getCollectionName()).thenReturn("opaque-transient-target"); + when(branches.findByIndexKind(RagBranchIndexKind.TRANSIENT)) + .thenReturn(List.of(index)); + when(generations.findByBranchIndexIdOrderByCreatedAtDesc(10L)) + .thenReturn(List.of(generation)); + when(pipeline.deleteBranch( + "workspace", "namespace", "release/candidate", + "opaque-transient-target")) + .thenReturn(true); + + new RagTransientBranchIndexCleanupService( + branches, generations, pipeline).cleanupExpired(); + + verify(branches).delete(index); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java index acc891ecc..46d092d58 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java @@ -7,8 +7,10 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -22,6 +24,9 @@ class RagPipelineClientTest { private RagPipelineClient client; private ObjectMapper objectMapper; + @TempDir + Path repositoryPath; + @BeforeEach void setUp() throws IOException { mockWebServer = new MockWebServer(); @@ -149,6 +154,57 @@ void testApplyChanges_DeleteOnlyOmitsRepositoryRoot() throws Exception { assertThat(payload).doesNotContainKey("repo_base"); } + @Test + @SuppressWarnings("unchecked") + void advanceGenerationBindsSourceAndTargetCollectionsAndRevisions() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("{\"generation_manifest_sha256\":\"digest\"}") + .addHeader("Content-Type", "application/json")); + + client.advanceGeneration( + List.of("src/Changed.java"), List.of("src/Deleted.java"), + "/tmp/repository", "workspace", "project", "develop", + "develop-400", "develop-401", + "a".repeat(64), + "opaque-generation-400", "opaque-generation-401"); + + RecordedRequest request = mockWebServer.takeRequest(); + assertThat(request.getPath()).isEqualTo("/index/advance-generation"); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload) + .containsEntry("source_commit", "develop-400") + .containsEntry("commit", "develop-401") + .containsEntry("source_tree_sha256", "a".repeat(64)) + .containsEntry("source_collection_target", "opaque-generation-400") + .containsEntry("collection_target", "opaque-generation-401") + .containsEntry("branch", "develop"); + assertThat((List) payload.get("updated_file_paths")) + .containsExactly("src/Changed.java"); + assertThat((List) payload.get("deleted_file_paths")) + .containsExactly("src/Deleted.java"); + } + + @Test + @SuppressWarnings("unchecked") + void exactGenerationPublicationRequestsReadableAliases() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("{\"generation_manifest_sha256\":\"digest\"}") + .addHeader("Content-Type", "application/json")); + + client.advanceGeneration( + List.of(), List.of(), "/tmp/repository", "workspace", "project", "main", + "main-400", "main-401", "a".repeat(64), + "opaque-generation-400", "opaque-generation-401", true, true); + + RecordedRequest request = mockWebServer.takeRequest(); + Map payload = objectMapper.readValue( + request.getBody().readUtf8(), Map.class); + assertThat(payload) + .containsEntry("publish_branch_alias", true) + .containsEntry("publish_legacy_project_alias", true); + } + @Test void testSemanticSearch_Success() throws Exception { Map mockResponse = Map.of( @@ -365,7 +421,7 @@ void testIndexRepository_Success() throws Exception { .addHeader("Content-Type", "application/json")); Map result = client.indexRepository( - "/tmp/repo", "ws", "proj", "main", "abc123", null, null); + repositoryPath.toString(), "ws", "proj", "main", "abc123", null, null); assertThat(result).containsEntry("document_count", 42); RecordedRequest request = mockWebServer.takeRequest(); @@ -382,7 +438,7 @@ void testIndexRepository_WithExcludePatterns() throws Exception { List patterns = List.of("*.log", "vendor/**"); Map result = client.indexRepository( - "/tmp/repo", "ws", "proj", "main", "abc123", null, patterns); + repositoryPath.toString(), "ws", "proj", "main", "abc123", null, patterns); assertThat(result).containsEntry("document_count", 30); RecordedRequest request = mockWebServer.takeRequest(); @@ -402,6 +458,30 @@ void testIndexRepository_WhenDisabled() throws Exception { assertThat(mockWebServer.getRequestCount()).isEqualTo(0); } + @Test + void testIndexRepository_StreamForwardsProgressAndReturnsTerminalResult() throws Exception { + mockWebServer.enqueue(new MockResponse() + .setBody("data: {\"type\":\"progress\",\"stage\":\"indexing\",\"indexedChunks\":5,\"estimatedChunks\":150,\"estimatedRemainingMs\":9000}\n\n" + + "data: {\"type\":\"complete\",\"result\":{\"document_count\":2,\"chunk_count\":5,\"generation_manifest_sha256\":\"digest\"}}\n\n") + .addHeader("Content-Type", "text/event-stream")); + List> progress = new ArrayList<>(); + + Map result = client.indexRepository( + repositoryPath.toString(), "ws", "proj", "develop", "abc123", + List.of("src/**"), List.of("vendor/**"), "generation-target", + progress::add); + + assertThat(progress).hasSize(1); + assertThat(progress.get(0)).containsEntry("indexedChunks", 5) + .containsEntry("estimatedChunks", 150) + .containsEntry("estimatedRemainingMs", 9000); + assertThat(result).containsEntry("chunk_count", 5) + .containsEntry("generation_manifest_sha256", "digest"); + RecordedRequest request = mockWebServer.takeRequest(); + assertThat(request.getPath()).endsWith("/index/repository/stream"); + assertThat(request.getHeader("Accept")).isEqualTo("text/event-stream"); + } + // ── deleteBranch tests ─────────────────────────────────────────────────── @Test diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java index 2d67645f0..ecb19a5ee 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java @@ -647,6 +647,46 @@ void testPerformIncrementalUpdate_BothDeletesAndUpdates() throws Exception { anyString(), eq("test-ws"), eq("test-proj"), eq("main"), eq("abc123")); } + @Test + void exactEmptyDeltaStillAdvancesImmutableRevision() throws Exception { + setupProjectWithWorkspace(); + when(branchArchiveService.downloadAndExtractSnapshotToDirectory( + any(), eq("ws-slug"), eq("repo-slug"), eq("develop-401"), + isNull(), any())) + .thenReturn(archiveSnapshot(Set.of(), Set.of())); + when(ragPipelineClient.advanceGeneration( + eq(List.of()), eq(List.of()), anyString(), + eq("test-ws"), eq("test-proj"), eq("develop"), + eq("develop-400"), eq("develop-401"), + anyString(), + eq("generation-400"), eq("generation-401"), + eq(true), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-401", + "document_count", 231, + "chunk_count", 400)); + + Map result = service.performIncrementalUpdate( + testProject, new VcsConnection(), "ws-slug", "repo-slug", + "develop", "develop-401", Set.of(), Set.of(), Set.of(), + "develop-400", "generation-400", "generation-401", + true, false); + + assertThat(result) + .containsEntry("status", "completed") + .containsEntry("generation_manifest_sha256", "manifest-401"); + verify(ragPipelineClient).advanceGeneration( + eq(List.of()), eq(List.of()), anyString(), + eq("test-ws"), eq("test-proj"), eq("develop"), + eq("develop-400"), eq("develop-401"), + anyString(), + eq("generation-400"), eq("generation-401"), + eq(true), eq(false)); + verify(ragPipelineClient, never()).applyChanges( + anyList(), anyList(), nullable(String.class), + anyString(), anyString(), anyString(), anyString()); + } + // ── Helpers ────────────────────────────────────────────────────────────── private void setupProjectWithWorkspace() { diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java new file mode 100644 index 000000000..4adc89b69 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java @@ -0,0 +1,179 @@ +package org.rostilos.codecrow.ragengine.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.*; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagIndexOperationRepository; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class RagBranchIndexRegistryServiceTest { + + @Mock + private RagBranchIndexRepository branchIndexRepository; + @Mock + private RagBranchIndexGenerationRepository generationRepository; + @Mock + private RagIndexOperationRepository operationRepository; + + private RagBranchIndexRegistryService service; + private Project project; + + @BeforeEach + void setUp() { + service = new RagBranchIndexRegistryService( + branchIndexRepository, generationRepository, operationRepository); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + when(branchIndexRepository.save(any())).thenAnswer(invocation -> { + RagBranchIndex value = invocation.getArgument(0); + if (value.getId() == null) { + value.setId(10L); + } + return value; + }); + when(generationRepository.save(any())).thenAnswer(invocation -> { + RagBranchIndexGeneration value = invocation.getArgument(0); + if (value.getId() == null) { + value.setId(20L); + } + return value; + }); + when(operationRepository.save(any())).thenAnswer(invocation -> { + RagIndexOperation value = invocation.getArgument(0); + if (value.getId() == null) { + value.setId(30L); + } + return value; + }); + } + + @Test + void registersIdempotentTenantScopedBuildWithoutLeakingBranchName() { + when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) + .thenReturn(Optional.empty()); + when(branchIndexRepository.findByProjectIdAndBranchName(42L, "client/private-develop")) + .thenReturn(Optional.empty()); + + var registration = service.registerBuild( + project, + "client/private-develop", + RagBranchIndexKind.DURABLE, + "master-100", + "develop-400", + "representation"); + + assertThat(registration.existingOperation()).isFalse(); + assertThat(registration.branchIndex().getIndexKind()).isEqualTo(RagBranchIndexKind.DURABLE); + assertThat(registration.branchIndex().getDesiredCommitHash()).isEqualTo("develop-400"); + assertThat(registration.generation().getSeedRevision()).isEqualTo("master-100"); + assertThat(registration.generation().getCollectionName()) + .startsWith("cc_w0_p42_b") + .doesNotContain("client", "private", "develop"); + assertThat(registration.operation().getOperationKey()).hasSize(64); + } + + @Test + void publishesNewGenerationAndSupersedesPreviousOneAtomically() { + RagBranchIndex branchIndex = new RagBranchIndex(project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + RagBranchIndexGeneration previous = new RagBranchIndexGeneration( + branchIndex, "develop-400", "generation-400", null, + "master-100", "representation"); + previous.setId(19L); + previous.activate("manifest-400", 500, 1500); + branchIndex.activate(previous); + when(branchIndexRepository.findByProjectIdAndBranchName(42L, "develop")) + .thenReturn(Optional.of(branchIndex)); + when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) + .thenReturn(Optional.empty()); + + var registration = service.registerBuild( + project, "develop", RagBranchIndexKind.DURABLE, + "develop-400", "develop-401", "representation"); + when(operationRepository.findById(30L)).thenReturn(Optional.of(registration.operation())); + + service.startBuild(30L, 99L); + RagBranchIndexGeneration published = service.publish(30L, "manifest-401", 501, 1504); + + assertThat(previous.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.SUPERSEDED); + assertThat(published.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.ACTIVE); + assertThat(branchIndex.getActiveGeneration()).isSameAs(published); + assertThat(branchIndex.getCommitHash()).isEqualTo("develop-401"); + assertThat(registration.operation().getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); + assertThat(registration.operation().getAttemptCount()).isEqualTo(1); + assertThat(registration.operation().getJobId()).isEqualTo(99L); + } + + @Test + void failedReplacementKeepsLastVerifiedGenerationAvailable() { + RagBranchIndex branchIndex = new RagBranchIndex(project, "master", RagBranchIndexKind.PRIMARY); + branchIndex.setId(10L); + RagBranchIndexGeneration active = new RagBranchIndexGeneration( + branchIndex, "master-100", "generation-100", null, + null, "representation"); + active.setId(19L); + active.activate("manifest-100", 500, 1500); + branchIndex.activate(active); + when(branchIndexRepository.findByProjectIdAndBranchName(42L, "master")) + .thenReturn(Optional.of(branchIndex)); + when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) + .thenReturn(Optional.empty()); + + var registration = service.registerBuild( + project, "master", RagBranchIndexKind.PRIMARY, + "master-100", "master-101", "representation"); + when(operationRepository.findById(30L)).thenReturn(Optional.of(registration.operation())); + + service.fail(30L, "vector publication failed"); + + assertThat(branchIndex.getActiveGeneration()).isSameAs(active); + assertThat(branchIndex.getCommitHash()).isEqualTo("master-100"); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("master-101"); + assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); + assertThat(registration.generation().getStatus()).isEqualTo(RagBranchIndexGenerationStatus.FAILED); + assertThat(registration.operation().getStatus()).isEqualTo(RagIndexOperationStatus.FAILED); + } + + @Test + void failedIdempotentOperationCanRetryWithoutCreatingDuplicateGeneration() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + RagBranchIndexGeneration generation = new RagBranchIndexGeneration( + branchIndex, "develop-401", "generation-401", null, + "develop-400", "representation"); + generation.setId(20L); + generation.fail("worker restarted"); + RagIndexOperation operation = new RagIndexOperation( + project, "develop", "develop-400", "develop-401", "key"); + operation.setId(30L); + operation.setGeneration(generation); + operation.fail("worker restarted"); + when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + + service.startBuild(30L, 91L); + + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.RUNNING); + assertThat(operation.getAttemptCount()).isEqualTo(1); + assertThat(operation.getJobId()).isEqualTo(91L); + assertThat(generation.getStatus()) + .isEqualTo(RagBranchIndexGenerationStatus.BUILDING); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-401"); + verify(generationRepository).save(generation); + verify(branchIndexRepository).save(branchIndex); + verify(operationRepository).save(operation); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java index 1c2045053..8956f29db 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java @@ -19,6 +19,7 @@ import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.core.service.AnalysisJobService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.springframework.test.util.ReflectionTestUtils; @@ -85,6 +86,62 @@ void testIsRagEnabled_ApiDisabled() { assertThat(result).isFalse(); } + @Test + void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Exception { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, registry, builder); + setupRagEnabled(); + setupVcsBinding(); + ReflectionTestUtils.setField(service, "ragApiEnabled", true); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + .thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag("")) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of(), Set.of(), Set.of())); + Job job = mock(Job.class); + when(job.getId()).thenReturn(77L); + when(analysisJobService.createRagIndexJob(any(), eq(false), any())) + .thenReturn(job); + when(analysisLockService.acquireLock( + eq(testProject), eq("feature"), any(), eq("develop-400"), isNull())) + .thenReturn(Optional.of("exact-feature-lock")); + VcsClient vcs = mock(VcsClient.class); + when(vcsClientProvider.getClient(any())).thenReturn(vcs); + when(vcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) + .thenReturn("develop-400"); + when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) + .thenReturn(Optional.empty()); + when(builder.build( + eq(testProject), any(), eq("my-workspace"), eq("my-repo"), + eq("feature"), eq("develop-400"), + eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), + any(), any(), eq(77L))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400, + "updatedFiles", 231, + "deletedFiles", 0, + "skippedFiles", 0)); + + boolean ready = service.ensureBranchIndexForPrTarget( + testProject, "feature", ignored -> { }); + + assertThat(ready).isTrue(); + verify(vcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); + verify(builder).build( + eq(testProject), any(), eq("my-workspace"), eq("my-repo"), + eq("feature"), eq("develop-400"), + eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), + any(), any(), eq(77L)); + } + @Test void testIsRagEnabled_NullConfig() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); @@ -749,6 +806,56 @@ void testUpdateBranchIndex_SuccessWithDiff() throws Exception { verify(mockVcs).getBranchDiff("my-workspace", "my-repo", "main", "feature"); } + @Test + void updateBranchIndexUsesCompletedBranchCheckpointWithoutComparingMain() throws Exception { + setupRagEnabled(); + setupVcsBinding(); + setupProjectWithWorkspaceAndNamespace(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); + when(incrementalRagUpdateService.parseDiffForRag("checkpoint diff")) + .thenReturn(new IncrementalRagUpdateService.DiffResult( + Set.of(), Set.of("src/Changed.java"), Set.of())); + when(analysisJobService.createRagIndexJob(eq(testProject), eq(false), any())).thenReturn(mock(Job.class)); + when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) + .thenReturn(Optional.of("lock")); + when(incrementalRagUpdateService.performIncrementalUpdate( + any(), any(), anyString(), anyString(), anyString(), anyString(), anySet(), anySet(), anySet())) + .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0, "skippedFiles", 0)); + + VcsClient mockVcs = mock(VcsClient.class); + when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); + when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("develop-401"); + when(mockVcs.getBranchDiff("my-workspace", "my-repo", "develop-400", "develop-401")) + .thenReturn("checkpoint diff"); + RagBranchIndex checkpoint = new RagBranchIndex(testProject, "feature"); + checkpoint.setCommitHash("develop-400"); + when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) + .thenReturn(Optional.of(checkpoint)); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); + + assertThat(result).isTrue(); + verify(mockVcs).getBranchDiff("my-workspace", "my-repo", "develop-400", "develop-401"); + verify(mockVcs, never()).getBranchDiff("my-workspace", "my-repo", "main", "feature"); + } + + @Test + void updateBranchIndexRejectsBranchOutsideRetainedConfiguration() { + setupRagEnabled(); + when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + boolean result = service.updateBranchIndex(testProject, "release/preview", eventConsumer); + + assertThat(result).isFalse(); + verifyNoInteractions(vcsClientProvider); + verify(eventConsumer).accept(argThat(event -> "rag_skipped".equals(event.get("state")))); + } + @Test void testUpdateBranchIndex_EmptyDiff() throws Exception { setupRagEnabled(); @@ -1123,7 +1230,8 @@ void testCleanupStaleBranches_DeleteThrows() throws Exception { private void setupRagEnabled() { ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(true, "main"); + RagConfig ragConfig = new RagConfig( + true, "main", null, null, true, 30, List.of("feature"), true); ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); testProject.setConfiguration(config); } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentityTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentityTest.java new file mode 100644 index 000000000..120039427 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/source/RepositorySourceTreeIdentityTest.java @@ -0,0 +1,50 @@ +package org.rostilos.codecrow.ragengine.source; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class RepositorySourceTreeIdentityTest { + + @TempDir + Path repository; + + @Test + void matchesTheRagConsumerCrossLanguageGoldenDigest() + throws Exception { + Files.createDirectories(repository.resolve("app")); + Files.writeString(repository.resolve("app/Module.php"), " + + 4.0.0 + + org.rostilos.codecrow + codecrow-parent + 1.0 + ../../pom.xml + + codecrow-scm-evidence + Provider-neutral commit evidence, promotion planning, and issue provenance + + + org.rostilos.codecrow + codecrow-vcs-client + + + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-autoconfigure + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-junit-jupiter + test + + + org.assertj + assertj-core + test + + + diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/module-info.java b/java-ecosystem/libs/scm-evidence/src/main/java/module-info.java new file mode 100644 index 000000000..9a0fbe016 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/module-info.java @@ -0,0 +1,25 @@ +module org.rostilos.codecrow.scmevidence { + requires jakarta.persistence; + requires spring.data.jpa; + requires spring.data.commons; + requires spring.context; + requires spring.boot.autoconfigure; + requires spring.tx; + requires org.slf4j; + requires org.rostilos.codecrow.vcs; + + exports org.rostilos.codecrow.scmevidence.api; + exports org.rostilos.codecrow.scmevidence.config; + exports org.rostilos.codecrow.scmevidence.model; + exports org.rostilos.codecrow.scmevidence.persistence; + exports org.rostilos.codecrow.scmevidence.service; + + opens org.rostilos.codecrow.scmevidence.model + to org.hibernate.orm.core, spring.core, spring.context; + opens org.rostilos.codecrow.scmevidence.persistence + to spring.core, spring.context; + opens org.rostilos.codecrow.scmevidence.service + to spring.core, spring.context; + opens org.rostilos.codecrow.scmevidence.config + to spring.core, spring.context; +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/AnalysisReceiptView.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/AnalysisReceiptView.java new file mode 100644 index 000000000..1a47e330b --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/AnalysisReceiptView.java @@ -0,0 +1,11 @@ +package org.rostilos.codecrow.scmevidence.api; + +public record AnalysisReceiptView( + String commitHash, + String patchId, + String sourceBranch, + String targetBranch, + String targetBaseRevision, + Long analysisId, + String analysisType) { +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/CommitEvidenceView.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/CommitEvidenceView.java new file mode 100644 index 000000000..dae1dc4e9 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/CommitEvidenceView.java @@ -0,0 +1,8 @@ +package org.rostilos.codecrow.scmevidence.api; + +public record CommitEvidenceView( + String commitHash, + String patchId, + String authorName, + String authorEmail) { +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/IssueProvenance.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/IssueProvenance.java new file mode 100644 index 000000000..7da75eb54 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/IssueProvenance.java @@ -0,0 +1,10 @@ +package org.rostilos.codecrow.scmevidence.api; + +public record IssueProvenance( + String commitHash, + String authorName, + String authorEmail, + String filePath, + int lineNumber, + String confidence) { +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/PromotionPlan.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/PromotionPlan.java new file mode 100644 index 000000000..786fed27e --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/api/PromotionPlan.java @@ -0,0 +1,16 @@ +package org.rostilos.codecrow.scmevidence.api; + +import java.util.List; + +public record PromotionPlan( + ReuseKind reuseKind, + List reusableCommits, + List commitsWithoutEvidence, + boolean requiresTargetContextAnalysis) { + + public enum ReuseKind { + EXACT_EVIDENCE_REUSE, + PARTIAL_EVIDENCE_REUSE, + NO_EVIDENCE_REUSE + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfiguration.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfiguration.java new file mode 100644 index 000000000..f66d35bcb --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfiguration.java @@ -0,0 +1,18 @@ +package org.rostilos.codecrow.scmevidence.config; + +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +/** + * Self-contained Spring Boot integration for the provider-neutral SCM evidence + * package. Applications opt in by including the package artifact; they do not + * need to know its internal service, persistence, or entity package layout. + */ +@AutoConfiguration +@ComponentScan(basePackages = "org.rostilos.codecrow.scmevidence.service") +@EnableJpaRepositories(basePackages = "org.rostilos.codecrow.scmevidence.persistence") +@EntityScan(basePackages = "org.rostilos.codecrow.scmevidence.model") +public class ScmEvidenceAutoConfiguration { +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAddedLineEvidence.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAddedLineEvidence.java new file mode 100644 index 000000000..1e39c88b9 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAddedLineEvidence.java @@ -0,0 +1,41 @@ +package org.rostilos.codecrow.scmevidence.model; + +import jakarta.persistence.*; + +@Entity +@Table(name = "scm_added_line_evidence", indexes = @Index( + name = "idx_scm_added_line_lookup", + columnList = "project_id,file_path,line_hash")) +public class ScmAddedLineEvidence { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "commit_evidence_id", nullable = false) + private ScmCommitEvidence commitEvidence; + @Column(name = "project_id", nullable = false) + private Long projectId; + @Column(name = "file_path", nullable = false, length = 1024) + private String filePath; + @Column(name = "new_line_number", nullable = false) + private int newLineNumber; + @Column(name = "line_hash", nullable = false, length = 64) + private String lineHash; + + public ScmAddedLineEvidence() {} + + public ScmAddedLineEvidence(Long projectId, String filePath, + int newLineNumber, String lineHash) { + this.projectId = projectId; + this.filePath = filePath; + this.newLineNumber = newLineNumber; + this.lineHash = lineHash; + } + + public void setCommitEvidence(ScmCommitEvidence commitEvidence) { + this.commitEvidence = commitEvidence; + } + public ScmCommitEvidence getCommitEvidence() { return commitEvidence; } + public String getFilePath() { return filePath; } + public int getNewLineNumber() { return newLineNumber; } + public String getLineHash() { return lineHash; } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAnalysisReceipt.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAnalysisReceipt.java new file mode 100644 index 000000000..d1ae427ba --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmAnalysisReceipt.java @@ -0,0 +1,55 @@ +package org.rostilos.codecrow.scmevidence.model; + +import jakarta.persistence.*; + +import java.time.OffsetDateTime; + +@Entity +@Table(name = "scm_analysis_receipt", uniqueConstraints = @UniqueConstraint( + name = "uq_scm_analysis_receipt_context", + columnNames = {"project_id", "commit_evidence_id", "context_key"})) +public class ScmAnalysisReceipt { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @Column(name = "project_id", nullable = false) + private Long projectId; + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "commit_evidence_id", nullable = false) + private ScmCommitEvidence commitEvidence; + @Column(name = "source_branch", length = 256) + private String sourceBranch; + @Column(name = "target_branch", nullable = false, length = 256) + private String targetBranch; + @Column(name = "target_base_revision", length = 64) + private String targetBaseRevision; + @Column(name = "analysis_id") + private Long analysisId; + @Column(name = "analysis_type", nullable = false, length = 40) + private String analysisType; + @Column(name = "context_key", nullable = false, length = 64) + private String contextKey; + @Column(name = "analyzed_at", nullable = false) + private OffsetDateTime analyzedAt = OffsetDateTime.now(); + + public ScmAnalysisReceipt() {} + + public ScmAnalysisReceipt(Long projectId, ScmCommitEvidence commitEvidence, + String sourceBranch, String targetBranch, String targetBaseRevision, + Long analysisId, String analysisType, String contextKey) { + this.projectId = projectId; + this.commitEvidence = commitEvidence; + this.sourceBranch = sourceBranch; + this.targetBranch = targetBranch; + this.targetBaseRevision = targetBaseRevision; + this.analysisId = analysisId; + this.analysisType = analysisType; + this.contextKey = contextKey; + } + + public ScmCommitEvidence getCommitEvidence() { return commitEvidence; } + public String getSourceBranch() { return sourceBranch; } + public String getTargetBranch() { return targetBranch; } + public String getTargetBaseRevision() { return targetBaseRevision; } + public Long getAnalysisId() { return analysisId; } + public String getAnalysisType() { return analysisType; } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmCommitEvidence.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmCommitEvidence.java new file mode 100644 index 000000000..07741a258 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/model/ScmCommitEvidence.java @@ -0,0 +1,56 @@ +package org.rostilos.codecrow.scmevidence.model; + +import jakarta.persistence.*; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; + +@Entity +@Table(name = "scm_commit_evidence", uniqueConstraints = @UniqueConstraint( + name = "uq_scm_commit_evidence_project_hash", + columnNames = {"project_id", "commit_hash"})) +public class ScmCommitEvidence { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @Column(name = "project_id", nullable = false) + private Long projectId; + @Column(name = "commit_hash", nullable = false, length = 64) + private String commitHash; + @Column(name = "patch_id", nullable = false, length = 64) + private String patchId; + @Column(name = "author_name", length = 200) + private String authorName; + @Column(name = "author_email", length = 320) + private String authorEmail; + @Column(name = "captured_at", nullable = false) + private OffsetDateTime capturedAt = OffsetDateTime.now(); + @OneToMany(mappedBy = "commitEvidence", cascade = CascadeType.ALL, + orphanRemoval = true) + private List addedLines = new ArrayList<>(); + + public ScmCommitEvidence() {} + + public ScmCommitEvidence(Long projectId, String commitHash, String patchId, + String authorName, String authorEmail) { + this.projectId = projectId; + this.commitHash = commitHash; + this.patchId = patchId; + this.authorName = authorName; + this.authorEmail = authorEmail; + } + + public void addLine(ScmAddedLineEvidence line) { + line.setCommitEvidence(this); + addedLines.add(line); + } + + public Long getId() { return id; } + public Long getProjectId() { return projectId; } + public String getCommitHash() { return commitHash; } + public String getPatchId() { return patchId; } + public String getAuthorName() { return authorName; } + public String getAuthorEmail() { return authorEmail; } + public OffsetDateTime getCapturedAt() { return capturedAt; } + public List getAddedLines() { return addedLines; } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAddedLineEvidenceRepository.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAddedLineEvidenceRepository.java new file mode 100644 index 000000000..e57645f34 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAddedLineEvidenceRepository.java @@ -0,0 +1,25 @@ +package org.rostilos.codecrow.scmevidence.persistence; + +import org.rostilos.codecrow.scmevidence.model.ScmAddedLineEvidence; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import java.util.List; + +public interface ScmAddedLineEvidenceRepository + extends JpaRepository { + @Query(""" + SELECT line FROM ScmAddedLineEvidence line + JOIN FETCH line.commitEvidence evidence + WHERE evidence.projectId = :projectId + AND evidence.commitHash IN :commitHashes + AND line.filePath = :filePath + AND line.lineHash = :lineHash + """) + List findMatchingLines( + @Param("projectId") Long projectId, + @Param("commitHashes") List commitHashes, + @Param("filePath") String filePath, + @Param("lineHash") String lineHash); +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAnalysisReceiptRepository.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAnalysisReceiptRepository.java new file mode 100644 index 000000000..66abee755 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmAnalysisReceiptRepository.java @@ -0,0 +1,14 @@ +package org.rostilos.codecrow.scmevidence.persistence; + +import org.rostilos.codecrow.scmevidence.model.ScmAnalysisReceipt; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; + +public interface ScmAnalysisReceiptRepository + extends JpaRepository { + boolean existsByProjectIdAndCommitEvidenceIdAndContextKey( + Long projectId, Long commitEvidenceId, String contextKey); + List findByProjectIdAndCommitEvidencePatchIdIn( + Long projectId, List patchIds); +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmCommitEvidenceRepository.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmCommitEvidenceRepository.java new file mode 100644 index 000000000..dcbe6cdb4 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/persistence/ScmCommitEvidenceRepository.java @@ -0,0 +1,17 @@ +package org.rostilos.codecrow.scmevidence.persistence; + +import org.rostilos.codecrow.scmevidence.model.ScmCommitEvidence; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.Optional; + +public interface ScmCommitEvidenceRepository + extends JpaRepository { + Optional findByProjectIdAndCommitHash( + Long projectId, String commitHash); + List findByProjectIdAndCommitHashIn( + Long projectId, List commitHashes); + List findByProjectIdAndPatchIdIn( + Long projectId, List patchIds); +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/PatchIdentity.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/PatchIdentity.java new file mode 100644 index 000000000..b072ef368 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/PatchIdentity.java @@ -0,0 +1,51 @@ +package org.rostilos.codecrow.scmevidence.service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** Stable identity for a patch across merge commits and cherry-picks. */ +public final class PatchIdentity { + private PatchIdentity() {} + + public static String sha256(String unifiedDiff) { + StringBuilder normalized = new StringBuilder(); + if (unifiedDiff != null) { + for (String raw : unifiedDiff.replace("\r\n", "\n").split("\n", -1)) { + String line = stripTrailingWhitespace(raw); + if (line.startsWith("diff --git ") + || line.startsWith("rename from ") + || line.startsWith("rename to ") + || line.startsWith("new file mode ") + || line.startsWith("deleted file mode ") + || (line.startsWith("+") && !line.startsWith("+++")) + || (line.startsWith("-") && !line.startsWith("---"))) { + normalized.append(line).append('\n'); + } + } + } + return digest(normalized.toString()); + } + + public static String lineSha256(String line) { + return digest(line == null ? "" : line.strip()); + } + + static String digest(String value) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 unavailable", impossible); + } + } + + private static String stripTrailingWhitespace(String line) { + int end = line.length(); + while (end > 0 && Character.isWhitespace(line.charAt(end - 1))) { + end--; + } + return line.substring(0, end); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceService.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceService.java new file mode 100644 index 000000000..8298761ee --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceService.java @@ -0,0 +1,253 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.rostilos.codecrow.scmevidence.api.AnalysisReceiptView; +import org.rostilos.codecrow.scmevidence.api.CommitEvidenceView; +import org.rostilos.codecrow.scmevidence.api.IssueProvenance; +import org.rostilos.codecrow.scmevidence.api.PromotionPlan; +import org.rostilos.codecrow.scmevidence.model.ScmAddedLineEvidence; +import org.rostilos.codecrow.scmevidence.model.ScmAnalysisReceipt; +import org.rostilos.codecrow.scmevidence.model.ScmCommitEvidence; +import org.rostilos.codecrow.scmevidence.persistence.ScmAddedLineEvidenceRepository; +import org.rostilos.codecrow.scmevidence.persistence.ScmAnalysisReceiptRepository; +import org.rostilos.codecrow.scmevidence.persistence.ScmCommitEvidenceRepository; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.model.VcsCommit; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Public boundary for provider-neutral SCM evidence. */ +@Service +public class ScmEvidenceService { + private final ScmCommitEvidenceRepository evidenceRepository; + private final ScmAnalysisReceiptRepository receiptRepository; + private final ScmAddedLineEvidenceRepository lineRepository; + private final ScmPromotionPlanner promotionPlanner; + private final UnifiedDiffAddedLineParser lineParser = + new UnifiedDiffAddedLineParser(); + + public ScmEvidenceService( + ScmCommitEvidenceRepository evidenceRepository, + ScmAnalysisReceiptRepository receiptRepository, + ScmAddedLineEvidenceRepository lineRepository, + ScmPromotionPlanner promotionPlanner) { + this.evidenceRepository = evidenceRepository; + this.receiptRepository = receiptRepository; + this.lineRepository = lineRepository; + this.promotionPlanner = promotionPlanner; + } + + @Transactional + public List capture( + Long projectId, + VcsClient client, + String workspace, + String repository, + List commits) throws IOException { + requireProject(projectId); + List result = new ArrayList<>(); + for (VcsCommit commit : commits == null ? List.of() : commits) { + Optional existing = evidenceRepository + .findByProjectIdAndCommitHash(projectId, commit.hash()); + ScmCommitEvidence evidence; + if (existing.isPresent()) { + evidence = existing.get(); + } else { + String diff = client.getCommitDiff( + workspace, repository, commit.hash()); + evidence = new ScmCommitEvidence( + projectId, + commit.hash(), + PatchIdentity.sha256(diff), + commit.authorName(), + commit.authorEmail()); + for (var line : lineParser.parse(diff)) { + evidence.addLine(new ScmAddedLineEvidence( + projectId, line.filePath(), line.lineNumber(), + line.lineHash())); + } + evidence = evidenceRepository.save(evidence); + } + result.add(view(evidence)); + } + return List.copyOf(result); + } + + @Transactional + public void recordAnalysisReceipts( + Long projectId, + List commitHashes, + String sourceBranch, + String targetBranch, + String targetBaseRevision, + Long analysisId, + String analysisType) { + requireProject(projectId); + if (commitHashes == null || commitHashes.isEmpty()) { + return; + } + String branch = requireText(targetBranch, "targetBranch"); + String type = requireText(analysisType, "analysisType"); + String contextKey = contextKey(branch, targetBaseRevision, type); + for (ScmCommitEvidence evidence : evidenceRepository + .findByProjectIdAndCommitHashIn(projectId, commitHashes)) { + if (!receiptRepository + .existsByProjectIdAndCommitEvidenceIdAndContextKey( + projectId, evidence.getId(), contextKey)) { + receiptRepository.save(new ScmAnalysisReceipt( + projectId, evidence, sourceBranch, branch, + targetBaseRevision, analysisId, type, contextKey)); + } + } + } + + @Transactional(readOnly = true) + public PromotionPlan planPromotion( + Long projectId, + List currentCommitHashes, + String targetBranch, + String targetBaseRevision) { + requireProject(projectId); + List current = evidenceRepository + .findByProjectIdAndCommitHashIn(projectId, currentCommitHashes); + Map byHash = new HashMap<>(); + current.forEach(evidence -> byHash.put(evidence.getCommitHash(), evidence)); + List ordered = currentCommitHashes.stream() + .map(hash -> byHash.containsKey(hash) + ? view(byHash.get(hash)) + : new CommitEvidenceView( + hash, "missing:" + hash, null, null)) + .toList(); + List patchIds = current.stream() + .map(ScmCommitEvidence::getPatchId) + .distinct() + .toList(); + List receipts = patchIds.isEmpty() + ? List.of() + : receiptRepository + .findByProjectIdAndCommitEvidencePatchIdIn( + projectId, patchIds) + .stream() + .map(ScmEvidenceService::view) + .toList(); + return promotionPlanner.plan( + ordered, receipts, requireText(targetBranch, "targetBranch"), + targetBaseRevision); + } + + @Transactional(readOnly = true) + public Optional resolveIssueProvenance( + Long projectId, + List commitHashesOldestFirst, + String filePath, + Integer lineNumber, + String codeSnippet) { + requireProject(projectId); + if (commitHashesOldestFirst == null + || commitHashesOldestFirst.isEmpty() + || filePath == null || codeSnippet == null + || codeSnippet.isBlank()) { + return Optional.empty(); + } + List snippetLines = snippetLines(codeSnippet, lineNumber); + List matches = new ArrayList<>(); + for (SnippetLine snippetLine : snippetLines) { + lineRepository.findMatchingLines( + projectId, commitHashesOldestFirst, filePath, + PatchIdentity.lineSha256(snippetLine.content())) + .forEach(line -> matches.add(new ProvenanceCandidate( + line, snippetLine.expectedLine()))); + } + Map order = new HashMap<>(); + for (int i = 0; i < commitHashesOldestFirst.size(); i++) { + order.put(commitHashesOldestFirst.get(i), i); + } + return matches.stream() + .min(Comparator + .comparingInt((ProvenanceCandidate candidate) -> + -order.getOrDefault( + candidate.line().getCommitEvidence() + .getCommitHash(), -1)) + .thenComparingInt(candidate -> candidate.expectedLine() == null + ? 0 + : Math.abs(candidate.line().getNewLineNumber() + - candidate.expectedLine()))) + .map(candidate -> new IssueProvenance( + candidate.line().getCommitEvidence().getCommitHash(), + candidate.line().getCommitEvidence().getAuthorName(), + candidate.line().getCommitEvidence().getAuthorEmail(), + candidate.line().getFilePath(), + candidate.line().getNewLineNumber(), + candidate.expectedLine() != null + && candidate.line().getNewLineNumber() + == candidate.expectedLine() + ? "EXACT_LINE_AND_CONTENT" + : "EXACT_CONTENT")); + } + + private static List snippetLines( + String codeSnippet, Integer firstLineNumber) { + String normalized = codeSnippet.replace("\r\n", "\n"); + String[] lines = normalized.split("\n", -1); + if (lines.length == 1) { + return List.of(new SnippetLine(lines[0], firstLineNumber)); + } + List result = new ArrayList<>(); + for (int index = 0; index < lines.length; index++) { + if (!lines[index].isBlank()) { + result.add(new SnippetLine( + lines[index], firstLineNumber == null + ? null : firstLineNumber + index)); + } + } + return result; + } + + private record SnippetLine(String content, Integer expectedLine) {} + + private record ProvenanceCandidate( + ScmAddedLineEvidence line, Integer expectedLine) {} + + private static CommitEvidenceView view(ScmCommitEvidence evidence) { + return new CommitEvidenceView( + evidence.getCommitHash(), evidence.getPatchId(), + evidence.getAuthorName(), evidence.getAuthorEmail()); + } + + private static AnalysisReceiptView view(ScmAnalysisReceipt receipt) { + ScmCommitEvidence evidence = receipt.getCommitEvidence(); + return new AnalysisReceiptView( + evidence.getCommitHash(), evidence.getPatchId(), + receipt.getSourceBranch(), receipt.getTargetBranch(), + receipt.getTargetBaseRevision(), receipt.getAnalysisId(), + receipt.getAnalysisType()); + } + + private static String contextKey( + String targetBranch, String targetBaseRevision, + String analysisType) { + return PatchIdentity.digest(targetBranch + "\n" + + (targetBaseRevision == null ? "" : targetBaseRevision) + + "\n" + analysisType); + } + + private static void requireProject(Long projectId) { + if (projectId == null) { + throw new IllegalArgumentException("projectId is required"); + } + } + + private static String requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " is required"); + } + return value.trim(); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlanner.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlanner.java new file mode 100644 index 000000000..ba45a2c4c --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlanner.java @@ -0,0 +1,46 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.rostilos.codecrow.scmevidence.api.AnalysisReceiptView; +import org.rostilos.codecrow.scmevidence.api.CommitEvidenceView; +import org.rostilos.codecrow.scmevidence.api.PromotionPlan; +import org.springframework.stereotype.Service; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@Service +public class ScmPromotionPlanner { + public PromotionPlan plan( + List currentCommits, + List priorReceipts, + String targetBranch, + String targetBaseRevision) { + Set knownPatchIds = new HashSet<>(); + Set exactContextPatchIds = new HashSet<>(); + for (AnalysisReceiptView receipt : priorReceipts) { + knownPatchIds.add(receipt.patchId()); + if (targetBranch.equals(receipt.targetBranch()) + && java.util.Objects.equals( + targetBaseRevision, receipt.targetBaseRevision())) { + exactContextPatchIds.add(receipt.patchId()); + } + } + List reusable = currentCommits.stream() + .filter(commit -> knownPatchIds.contains(commit.patchId())) + .map(CommitEvidenceView::commitHash) + .toList(); + List unseen = currentCommits.stream() + .filter(commit -> !knownPatchIds.contains(commit.patchId())) + .map(CommitEvidenceView::commitHash) + .toList(); + PromotionPlan.ReuseKind kind = reusable.isEmpty() + ? PromotionPlan.ReuseKind.NO_EVIDENCE_REUSE + : unseen.isEmpty() + ? PromotionPlan.ReuseKind.EXACT_EVIDENCE_REUSE + : PromotionPlan.ReuseKind.PARTIAL_EVIDENCE_REUSE; + boolean contextAnalysisRequired = currentCommits.stream() + .anyMatch(commit -> !exactContextPatchIds.contains(commit.patchId())); + return new PromotionPlan(kind, reusable, unseen, contextAnalysisRequired); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParser.java b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParser.java new file mode 100644 index 000000000..95f865a2e --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParser.java @@ -0,0 +1,43 @@ +package org.rostilos.codecrow.scmevidence.service; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +final class UnifiedDiffAddedLineParser { + private static final Pattern TARGET_FILE = Pattern.compile("^\\+\\+\\+ b/(.+)$"); + private static final Pattern HUNK = Pattern.compile( + "^@@ -\\d+(?:,\\d+)? \\+(\\d+)(?:,(\\d+))? @@.*$"); + + record AddedLine(String filePath, int lineNumber, String lineHash) {} + + List parse(String diff) { + List result = new ArrayList<>(); + String file = null; + int newLine = -1; + for (String line : (diff == null ? "" : diff.replace("\r\n", "\n")).split("\n", -1)) { + Matcher fileMatcher = TARGET_FILE.matcher(line); + if (fileMatcher.matches()) { + file = fileMatcher.group(1); + continue; + } + Matcher hunkMatcher = HUNK.matcher(line); + if (hunkMatcher.matches()) { + newLine = Integer.parseInt(hunkMatcher.group(1)); + continue; + } + if (file == null || newLine < 0 || line.startsWith("\\ No newline")) { + continue; + } + if (line.startsWith("+") && !line.startsWith("+++")) { + result.add(new AddedLine(file, newLine, + PatchIdentity.lineSha256(line.substring(1)))); + newLine++; + } else if (!line.startsWith("-")) { + newLine++; + } + } + return result; + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/java-ecosystem/libs/scm-evidence/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 000000000..69da54961 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +org.rostilos.codecrow.scmevidence.config.ScmEvidenceAutoConfiguration diff --git a/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfigurationTest.java b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfigurationTest.java new file mode 100644 index 000000000..7fbb3a611 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/config/ScmEvidenceAutoConfigurationTest.java @@ -0,0 +1,33 @@ +package org.rostilos.codecrow.scmevidence.config; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +import static org.assertj.core.api.Assertions.assertThat; + +class ScmEvidenceAutoConfigurationTest { + + @Test + void ownsItsSpringIntegrationBoundaries() { + assertThat(ScmEvidenceAutoConfiguration.class) + .hasAnnotation(AutoConfiguration.class); + + ComponentScan componentScan = ScmEvidenceAutoConfiguration.class + .getAnnotation(ComponentScan.class); + assertThat(componentScan.basePackages()) + .containsExactly("org.rostilos.codecrow.scmevidence.service"); + + EnableJpaRepositories repositories = ScmEvidenceAutoConfiguration.class + .getAnnotation(EnableJpaRepositories.class); + assertThat(repositories.basePackages()) + .containsExactly("org.rostilos.codecrow.scmevidence.persistence"); + + EntityScan entities = ScmEvidenceAutoConfiguration.class + .getAnnotation(EntityScan.class); + assertThat(entities.basePackages()) + .containsExactly("org.rostilos.codecrow.scmevidence.model"); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceServiceTest.java b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceServiceTest.java new file mode 100644 index 000000000..61122c7ab --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmEvidenceServiceTest.java @@ -0,0 +1,142 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.scmevidence.model.ScmAddedLineEvidence; +import org.rostilos.codecrow.scmevidence.model.ScmCommitEvidence; +import org.rostilos.codecrow.scmevidence.persistence.ScmAddedLineEvidenceRepository; +import org.rostilos.codecrow.scmevidence.persistence.ScmAnalysisReceiptRepository; +import org.rostilos.codecrow.scmevidence.persistence.ScmCommitEvidenceRepository; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.model.VcsCommit; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class ScmEvidenceServiceTest { + @Mock private ScmCommitEvidenceRepository evidenceRepository; + @Mock private ScmAnalysisReceiptRepository receiptRepository; + @Mock private ScmAddedLineEvidenceRepository lineRepository; + @Mock private VcsClient vcsClient; + + private ScmEvidenceService service; + + @BeforeEach + void setUp() { + service = new ScmEvidenceService( + evidenceRepository, receiptRepository, lineRepository, + new ScmPromotionPlanner()); + } + + @Test + void capturesProviderNeutralPatchAuthorAndAddedLineEvidence() throws Exception { + VcsCommit commit = new VcsCommit( + "commit-1", "message", "Actual Author", "author@example.test", + OffsetDateTime.now(), List.of("parent")); + when(evidenceRepository.findByProjectIdAndCommitHash(42L, "commit-1")) + .thenReturn(Optional.empty()); + when(vcsClient.getCommitDiff("workspace", "repo", "commit-1")) + .thenReturn(""" + diff --git a/src/A.java b/src/A.java + --- a/src/A.java + +++ b/src/A.java + @@ -9,0 +10,1 @@ + +return secure(value); + """); + when(evidenceRepository.save(any())).thenAnswer(invocation -> invocation.getArgument(0)); + + var views = service.capture( + 42L, vcsClient, "workspace", "repo", List.of(commit)); + + assertThat(views).singleElement().satisfies(view -> { + assertThat(view.commitHash()).isEqualTo("commit-1"); + assertThat(view.patchId()).hasSize(64); + assertThat(view.authorName()).isEqualTo("Actual Author"); + assertThat(view.authorEmail()).isEqualTo("author@example.test"); + }); + var evidence = org.mockito.ArgumentCaptor.forClass(ScmCommitEvidence.class); + verify(evidenceRepository).save(evidence.capture()); + assertThat(evidence.getValue().getAddedLines()).singleElement() + .satisfies(line -> { + assertThat(line.getFilePath()).isEqualTo("src/A.java"); + assertThat(line.getNewLineNumber()).isEqualTo(10); + assertThat(line.getLineHash()) + .isEqualTo(PatchIdentity.lineSha256("return secure(value);")); + }); + } + + @Test + void issueProvenanceReturnsCommitAuthorNotReviewCommentAuthor() { + ScmCommitEvidence evidence = new ScmCommitEvidence( + 42L, "introducing-commit", "a".repeat(64), + "Actual Commit Author", "author@example.test"); + ScmAddedLineEvidence line = new ScmAddedLineEvidence( + 42L, "src/A.java", 10, + PatchIdentity.lineSha256("return secure(value);")); + line.setCommitEvidence(evidence); + when(lineRepository.findMatchingLines( + 42L, List.of("introducing-commit"), "src/A.java", + PatchIdentity.lineSha256("return secure(value);"))) + .thenReturn(List.of(line)); + + var provenance = service.resolveIssueProvenance( + 42L, List.of("introducing-commit"), "src/A.java", 10, + "return secure(value);"); + + assertThat(provenance).isPresent(); + assertThat(provenance.orElseThrow().authorName()) + .isEqualTo("Actual Commit Author"); + assertThat(provenance.orElseThrow().confidence()) + .isEqualTo("EXACT_LINE_AND_CONTENT"); + } + + @Test + void issueProvenanceResolvesAnAddedLineInsideMultilineSnippet() { + ScmCommitEvidence evidence = new ScmCommitEvidence( + 42L, "introducing-commit", "a".repeat(64), + "Actual Commit Author", "author@example.test"); + ScmAddedLineEvidence line = new ScmAddedLineEvidence( + 42L, "src/A.java", 11, + PatchIdentity.lineSha256("return secure(value);")); + line.setCommitEvidence(evidence); + when(lineRepository.findMatchingLines( + 42L, List.of("introducing-commit"), "src/A.java", + PatchIdentity.lineSha256("if (allowed) {"))) + .thenReturn(List.of()); + when(lineRepository.findMatchingLines( + 42L, List.of("introducing-commit"), "src/A.java", + PatchIdentity.lineSha256("return secure(value);"))) + .thenReturn(List.of(line)); + when(lineRepository.findMatchingLines( + 42L, List.of("introducing-commit"), "src/A.java", + PatchIdentity.lineSha256("}"))) + .thenReturn(List.of()); + + var provenance = service.resolveIssueProvenance( + 42L, List.of("introducing-commit"), "src/A.java", 10, + "if (allowed) {\nreturn secure(value);\n}"); + + assertThat(provenance).isPresent(); + assertThat(provenance.orElseThrow().lineNumber()).isEqualTo(11); + assertThat(provenance.orElseThrow().confidence()) + .isEqualTo("EXACT_LINE_AND_CONTENT"); + } + + @Test + void tenantProjectIdIsMandatoryAtPublicBoundary() { + assertThat(org.assertj.core.api.Assertions.catchThrowable(() -> + service.capture(null, vcsClient, "workspace", "repo", List.of()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("projectId"); + verifyNoInteractions(evidenceRepository, receiptRepository, lineRepository); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlannerTest.java b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlannerTest.java new file mode 100644 index 000000000..2f5c363d8 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/ScmPromotionPlannerTest.java @@ -0,0 +1,109 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.scmevidence.api.AnalysisReceiptView; +import org.rostilos.codecrow.scmevidence.api.CommitEvidenceView; +import org.rostilos.codecrow.scmevidence.api.PromotionPlan; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +class ScmPromotionPlannerTest { + private final ScmPromotionPlanner planner = new ScmPromotionPlanner(); + + @Test + void patchIdentitySurvivesCherryPickCoordinatesButNotContentChanges() { + String first = """ + diff --git a/src/A.java b/src/A.java + index 1111111..2222222 100644 + --- a/src/A.java + +++ b/src/A.java + @@ -10,2 +10,3 @@ + context + +return secure(value); + """; + String cherryPicked = """ + diff --git a/src/A.java b/src/A.java + index aaaaaaa..bbbbbbb 100644 + --- a/src/A.java + +++ b/src/A.java + @@ -410,2 +512,3 @@ + different surrounding context + +return secure(value); + """; + String changed = cherryPicked.replace("secure(value)", "unsafe(value)"); + + assertThat(PatchIdentity.sha256(cherryPicked)) + .isEqualTo(PatchIdentity.sha256(first)); + assertThat(PatchIdentity.sha256(changed)) + .isNotEqualTo(PatchIdentity.sha256(first)); + } + + @Test + void developToMasterReusesFourHundredCommitEvidenceButReviewsMasterContext() { + List commits = IntStream.range(0, 400) + .mapToObj(index -> commit(index, "patch-" + index)) + .toList(); + // The 400 commits deliberately rotate through the 231-file fixture. + assertThat(IntStream.range(0, 400) + .map(index -> index % 231).distinct().count()).isEqualTo(231); + List developReceipts = commits.stream() + .map(commit -> receipt(commit, "feature/1-x", "develop", "dev-base")) + .toList(); + + PromotionPlan master = planner.plan( + commits, developReceipts, "master", "master-base"); + + assertThat(master.reuseKind()) + .isEqualTo(PromotionPlan.ReuseKind.EXACT_EVIDENCE_REUSE); + assertThat(master.reusableCommits()).hasSize(400); + assertThat(master.commitsWithoutEvidence()).isEmpty(); + assertThat(master.requiresTargetContextAnalysis()).isTrue(); + + PromotionPlan exactDevelopRetry = planner.plan( + commits, developReceipts, "develop", "dev-base"); + assertThat(exactDevelopRetry.requiresTargetContextAnalysis()).isFalse(); + } + + @Test + void promotionWithAdditionalReleaseCommitsIsPartialAndRequiresAnalysis() { + List develop = IntStream.range(0, 12) + .mapToObj(index -> commit(index, "patch-" + index)) + .toList(); + List promoted = new ArrayList<>(develop); + promoted.add(commit(12, "release-hotfix")); + promoted.add(commit(13, "release-metadata")); + List receipts = develop.stream() + .map(commit -> receipt(commit, "feature", "develop", "d1")) + .toList(); + + PromotionPlan plan = planner.plan( + promoted, receipts, "master", "m1"); + + assertThat(plan.reuseKind()) + .isEqualTo(PromotionPlan.ReuseKind.PARTIAL_EVIDENCE_REUSE); + assertThat(plan.reusableCommits()).hasSize(12); + assertThat(plan.commitsWithoutEvidence()) + .containsExactly("commit-12", "commit-13"); + assertThat(plan.requiresTargetContextAnalysis()).isTrue(); + } + + private static CommitEvidenceView commit(int index, String patch) { + return new CommitEvidenceView( + "commit-" + index, patch, "author-" + index, + "author-" + index + "@example.test"); + } + + private static AnalysisReceiptView receipt( + CommitEvidenceView commit, + String source, + String target, + String base) { + return new AnalysisReceiptView( + commit.commitHash(), commit.patchId(), source, target, base, + 100L, "PR_REVIEW"); + } +} diff --git a/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParserTest.java b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParserTest.java new file mode 100644 index 000000000..a95aa34f7 --- /dev/null +++ b/java-ecosystem/libs/scm-evidence/src/test/java/org/rostilos/codecrow/scmevidence/service/UnifiedDiffAddedLineParserTest.java @@ -0,0 +1,34 @@ +package org.rostilos.codecrow.scmevidence.service; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class UnifiedDiffAddedLineParserTest { + @Test + void capturesRenamedTargetPathAndExactNewLineCoordinates() { + String diff = """ + diff --git a/old/A.java b/new/A.java + similarity index 80% + rename from old/A.java + rename to new/A.java + --- a/old/A.java + +++ b/new/A.java + @@ -7,2 +20,4 @@ + keep(); + -old(); + +introducedByAlice(); + +secondLine(); + tail(); + """; + + var lines = new UnifiedDiffAddedLineParser().parse(diff); + + assertThat(lines).hasSize(2); + assertThat(lines.get(0).filePath()).isEqualTo("new/A.java"); + assertThat(lines.get(0).lineNumber()).isEqualTo(21); + assertThat(lines.get(0).lineHash()) + .isEqualTo(PatchIdentity.lineSha256("introducedByAlice();")); + assertThat(lines.get(1).lineNumber()).isEqualTo(22); + } +} diff --git a/java-ecosystem/pom.xml b/java-ecosystem/pom.xml index 5c827ea26..51123c1f9 100644 --- a/java-ecosystem/pom.xml +++ b/java-ecosystem/pom.xml @@ -116,6 +116,12 @@ 1.0 + + org.rostilos.codecrow + codecrow-scm-evidence + 1.0 + + org.rostilos.codecrow codecrow-file-content @@ -582,6 +588,7 @@ libs/vcs-client libs/core + libs/scm-evidence libs/commit-graph libs/file-content libs/security diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/RagIndexingController.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/RagIndexingController.java index 4a87510f2..cb26ed218 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/RagIndexingController.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/RagIndexingController.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.rostilos.codecrow.core.dto.project.ProjectDTO; -import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; -import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; import org.rostilos.codecrow.ragengine.service.VcsRagIndexingService; +import org.rostilos.codecrow.ragengine.branch.BranchIndexMaintenanceService; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; @@ -32,19 +32,19 @@ public class RagIndexingController { private static final String EOF_MARKER = "__EOF__"; private final VcsRagIndexingService vcsRagIndexingService; - private final RagIndexTrackingService ragIndexTrackingService; - private final AnalysisLockService analysisLockService; + private final BranchIndexMaintenanceService branchIndexMaintenanceService; + private final ProjectRepository projectRepository; private final ObjectMapper objectMapper; public RagIndexingController( VcsRagIndexingService vcsRagIndexingService, - RagIndexTrackingService ragIndexTrackingService, - AnalysisLockService analysisLockService, + BranchIndexMaintenanceService branchIndexMaintenanceService, + ProjectRepository projectRepository, ObjectMapper objectMapper ) { this.vcsRagIndexingService = vcsRagIndexingService; - this.ragIndexTrackingService = ragIndexTrackingService; - this.analysisLockService = analysisLockService; + this.branchIndexMaintenanceService = branchIndexMaintenanceService; + this.projectRepository = projectRepository; this.objectMapper = objectMapper; } @@ -73,11 +73,14 @@ public ResponseEntity triggerIndexing( CompletableFuture> indexingFuture = CompletableFuture.supplyAsync(() -> { try { - return vcsRagIndexingService.indexProjectFromVcs( - authProject, - request.branch(), - messageConsumer - ); + if (request.allConfiguredBranches() + || (request.branch() != null && !request.branch().isBlank())) { + var project = projectRepository.findByIdWithFullDetails(authProject.id()) + .orElseThrow(() -> new IllegalStateException("Project not found")); + return branchIndexMaintenanceService.rebuild( + project, request.branch(), request.allConfiguredBranches(), messageConsumer); + } + return vcsRagIndexingService.indexProjectFromVcs(authProject, null, messageConsumer); } catch (Exception e) { log.error("RAG indexing failed", e); return Map.of( @@ -144,6 +147,7 @@ public ResponseEntity canStartIndexing(@PathVariable Long projectId) { } public record RagIndexRequest( - String branch // Optional: branch to index. If null, uses project's configured RAG branch or default + String branch, + boolean allConfiguredBranches ) {} } diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/controller/ProjectController.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/controller/ProjectController.java index 9e17203a8..4ced2b305 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/controller/ProjectController.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/controller/ProjectController.java @@ -29,11 +29,13 @@ import org.rostilos.codecrow.webserver.project.dto.request.ChangeVcsConnectionRequest; import org.rostilos.codecrow.webserver.project.dto.ProjectTokenDTO; import org.rostilos.codecrow.webserver.project.dto.response.RagIndexStatusDTO; +import org.rostilos.codecrow.webserver.project.dto.response.RagBranchIndexStatusDTO; import org.rostilos.codecrow.webserver.auth.service.TwoFactorAuthService; import org.rostilos.codecrow.webserver.project.service.ProjectService; import org.rostilos.codecrow.webserver.project.service.ProjectTokenService; import org.rostilos.codecrow.webserver.project.service.RagIndexStatusService; import org.rostilos.codecrow.webserver.project.service.RagIndexingTriggerService; +import org.rostilos.codecrow.webserver.project.service.RagBranchIndexStatusService; import org.rostilos.codecrow.webserver.project.service.VectorStorageService; import org.rostilos.codecrow.webserver.workspace.service.WorkspaceService; import org.springframework.http.HttpStatus; @@ -66,6 +68,7 @@ public class ProjectController { private final WorkspaceService workspaceService; private final RagIndexStatusService ragIndexStatusService; private final RagIndexingTriggerService ragIndexingTriggerService; + private final RagBranchIndexStatusService ragBranchIndexStatusService; private final VectorStorageService vectorStorageService; private final TwoFactorAuthService twoFactorAuthService; @@ -75,6 +78,7 @@ public ProjectController( WorkspaceService workspaceService, RagIndexStatusService ragIndexStatusService, RagIndexingTriggerService ragIndexingTriggerService, + RagBranchIndexStatusService ragBranchIndexStatusService, VectorStorageService vectorStorageService, TwoFactorAuthService twoFactorAuthService) { this.projectService = projectService; @@ -82,6 +86,7 @@ public ProjectController( this.workspaceService = workspaceService; this.ragIndexStatusService = ragIndexStatusService; this.ragIndexingTriggerService = ragIndexingTriggerService; + this.ragBranchIndexStatusService = ragBranchIndexStatusService; this.vectorStorageService = vectorStorageService; this.twoFactorAuthService = twoFactorAuthService; } @@ -416,6 +421,20 @@ public ResponseEntity getRagIndexStatus( ragIndexStatusService.canStartIndexing(project)), HttpStatus.OK); } + /** + * Returns a stable, tenant-scoped operational view of the configured + * primary and retained RAG branches. It never exposes PR-only transient + * snapshots or physical vector collection names. + */ + @GetMapping("/{projectNamespace}/rag/branches") + public ResponseEntity> getRagBranchIndexes( + @PathVariable String workspaceSlug, + @PathVariable String projectNamespace) { + Workspace workspace = workspaceService.getWorkspaceBySlug(workspaceSlug); + Project project = projectService.getProjectByWorkspaceAndNamespace(workspace.getId(), projectNamespace); + return ResponseEntity.ok(ragBranchIndexStatusService.getConfiguredBranches(project)); + } + /** * PUT /api/workspace/{workspaceSlug}/project/{projectNamespace}/rag/config * Updates the RAG configuration for the project (enable/disable, set branch, @@ -437,7 +456,9 @@ public ResponseEntity updateRagConfig( request.getIncludePatterns(), request.getExcludePatterns(), request.getMultiBranchEnabled(), - request.getBranchRetentionDays()); + request.getBranchRetentionDays(), + request.getIndexedBranches(), + request.getTransientBranchIndexesEnabled()); return new ResponseEntity<>(ProjectDTO.fromProject(updated), HttpStatus.OK); } @@ -457,6 +478,7 @@ public SseEmitter triggerRagIndexing( @PathVariable String workspaceSlug, @PathVariable String projectNamespace, @RequestParam(required = false) String branch, + @RequestParam(required = false, defaultValue = "false") boolean allConfiguredBranches, @AuthenticationPrincipal UserDetailsImpl userDetails) { Workspace workspace = workspaceService.getWorkspaceBySlug(workspaceSlug); Project project = projectService.getProjectByWorkspaceAndNamespace(workspace.getId(), projectNamespace); @@ -494,6 +516,7 @@ public SseEmitter triggerRagIndexing( project.getId(), userDetails.getId(), branch, + allConfiguredBranches, emitter); return emitter; diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/request/UpdateRagConfigRequest.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/request/UpdateRagConfigRequest.java index 84c564430..96410dd35 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/request/UpdateRagConfigRequest.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/request/UpdateRagConfigRequest.java @@ -18,6 +18,10 @@ public class UpdateRagConfigRequest { private Integer branchRetentionDays; + private List indexedBranches; + + private Boolean transientBranchIndexesEnabled; + public UpdateRagConfigRequest() { } @@ -34,13 +38,23 @@ public UpdateRagConfigRequest(Boolean enabled, String branch, List exclu public UpdateRagConfigRequest(Boolean enabled, String branch, List includePatterns, List excludePatterns, - Boolean multiBranchEnabled, Integer branchRetentionDays) { + Boolean multiBranchEnabled, Integer branchRetentionDays, + List indexedBranches, Boolean transientBranchIndexesEnabled) { this.enabled = enabled; this.branch = branch; this.includePatterns = includePatterns; this.excludePatterns = excludePatterns; this.multiBranchEnabled = multiBranchEnabled; this.branchRetentionDays = branchRetentionDays; + this.indexedBranches = indexedBranches; + this.transientBranchIndexesEnabled = transientBranchIndexesEnabled; + } + + public UpdateRagConfigRequest(Boolean enabled, String branch, List includePatterns, + List excludePatterns, + Boolean multiBranchEnabled, Integer branchRetentionDays) { + this(enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, + branchRetentionDays, null, null); } public Boolean getEnabled() { @@ -90,4 +104,20 @@ public Integer getBranchRetentionDays() { public void setBranchRetentionDays(Integer branchRetentionDays) { this.branchRetentionDays = branchRetentionDays; } + + public List getIndexedBranches() { + return indexedBranches; + } + + public void setIndexedBranches(List indexedBranches) { + this.indexedBranches = indexedBranches; + } + + public Boolean getTransientBranchIndexesEnabled() { + return transientBranchIndexesEnabled; + } + + public void setTransientBranchIndexesEnabled(Boolean transientBranchIndexesEnabled) { + this.transientBranchIndexesEnabled = transientBranchIndexesEnabled; + } } diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/response/RagBranchIndexStatusDTO.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/response/RagBranchIndexStatusDTO.java new file mode 100644 index 000000000..16e42ecd4 --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/dto/response/RagBranchIndexStatusDTO.java @@ -0,0 +1,19 @@ +package org.rostilos.codecrow.webserver.project.dto.response; + +import java.time.OffsetDateTime; + +/** + * Observable state of one configured RAG branch. Physical collection names are + * deliberately not exposed: they are internal, tenant-scoped storage details. + */ +public record RagBranchIndexStatusDTO( + String branchName, + String role, + String status, + String activeRevision, + String requestedRevision, + Integer fileCount, + Integer chunkCount, + OffsetDateTime lastUpdatedAt, + String errorMessage) { +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/IProjectService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/IProjectService.java index b31f14f99..ff1ad28a1 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/IProjectService.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/IProjectService.java @@ -76,6 +76,11 @@ void updateRepositorySettings(Long workspaceId, Long projectId, UpdateRepository Project updateBranchAnalysisConfig(Long workspaceId, Long projectId, List prTargetBranches, List branchPushPatterns); + Project updateRagConfig(Long workspaceId, Long projectId, boolean enabled, String branch, + List includePatterns, + List excludePatterns, Boolean multiBranchEnabled, Integer branchRetentionDays, + List indexedBranches, Boolean transientBranchIndexesEnabled); + Project updateRagConfig(Long workspaceId, Long projectId, boolean enabled, String branch, List includePatterns, List excludePatterns, Boolean multiBranchEnabled, Integer branchRetentionDays); diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java index eca688de0..31f4d689f 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java @@ -740,7 +740,9 @@ public Project updateRagConfig( java.util.List includePatterns, java.util.List excludePatterns, Boolean multiBranchEnabled, - Integer branchRetentionDays) { + Integer branchRetentionDays, + java.util.List indexedBranches, + Boolean transientBranchIndexesEnabled) { Project project = projectRepository.findByWorkspaceIdAndId(workspaceId, projectId) .orElseThrow(() -> new NoSuchElementException("Project not found")); @@ -758,7 +760,8 @@ public Project updateRagConfig( var commentCommands = currentConfig != null ? currentConfig.commentCommands() : null; RagConfig ragConfig = new RagConfig( - enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, branchRetentionDays); + enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, branchRetentionDays, + indexedBranches, transientBranchIndexesEnabled); ProjectConfig newConfig = new ProjectConfig(useLocalMcp, useMcpTools, mainBranch, branchAnalysis, ragConfig, prAnalysisEnabled, branchAnalysisEnabled, installationMethod, commentCommands, @@ -768,6 +771,20 @@ public Project updateRagConfig( return projectRepository.save(project); } + @Transactional + public Project updateRagConfig( + Long workspaceId, + Long projectId, + boolean enabled, + String branch, + java.util.List includePatterns, + java.util.List excludePatterns, + Boolean multiBranchEnabled, + Integer branchRetentionDays) { + return updateRagConfig(workspaceId, projectId, enabled, branch, includePatterns, excludePatterns, + multiBranchEnabled, branchRetentionDays, null, null); + } + /** * Simplified RAG config update (backward compatible). */ @@ -779,7 +796,8 @@ public Project updateRagConfig( String branch, java.util.List includePatterns, java.util.List excludePatterns) { - return updateRagConfig(workspaceId, projectId, enabled, branch, includePatterns, excludePatterns, null, null); + return updateRagConfig(workspaceId, projectId, enabled, branch, includePatterns, excludePatterns, + null, null, null, null); } @Transactional @@ -976,7 +994,8 @@ public Project syncAnalysisScope(Long workspaceId, Long projectId, String direct RagConfig rag = config.ragConfig() != null ? config.ragConfig() : new RagConfig(); config.setRagConfig(new RagConfig( rag.enabled(), rag.branch(), scope.includePatterns(), scope.excludePatterns(), - rag.multiBranchEnabled(), rag.branchRetentionDays())); + rag.multiBranchEnabled(), rag.branchRetentionDays(), rag.indexedBranches(), + rag.transientBranchIndexesEnabled())); } else { throw new IllegalArgumentException("direction must be FROM_RAG or TO_RAG"); } diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusService.java new file mode 100644 index 000000000..dd61cc3d0 --- /dev/null +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusService.java @@ -0,0 +1,127 @@ +package org.rostilos.codecrow.webserver.project.service; + +import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; +import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.webserver.project.dto.response.RagBranchIndexStatusDTO; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Read-only projection of the primary and explicitly retained RAG branches. + * It intentionally excludes transient PR-only snapshots from the configuration + * view, while retaining a compatible primary status for pre-generation projects. + */ +@Service +public class RagBranchIndexStatusService { + private final RagBranchIndexRepository branchIndexRepository; + private final RagIndexStatusService projectStatusService; + + public RagBranchIndexStatusService( + RagBranchIndexRepository branchIndexRepository, + RagIndexStatusService projectStatusService) { + this.branchIndexRepository = branchIndexRepository; + this.projectStatusService = projectStatusService; + } + + @Transactional(readOnly = true) + public List getConfiguredBranches(Project project) { + if (project.getConfiguration() == null || project.getConfiguration().ragConfig() == null) { + return List.of(); + } + var config = project.getConfiguration().ragConfig(); + String primary = resolvePrimary(project, config.branch()); + if (primary == null) { + return List.of(); + } + + Map persisted = new LinkedHashMap<>(); + for (RagBranchIndex index : branchIndexRepository.findByProjectId(project.getId())) { + persisted.put(index.getBranchName(), index); + } + RagIndexStatus projectStatus = projectStatusService.getIndexStatus(project).orElse(null); + + List result = new ArrayList<>(); + result.add(toDto(primary, "PRIMARY", persisted.get(primary), projectStatus)); + for (String branch : config.getEffectiveIndexedBranches()) { + if (!primary.equals(branch)) { + result.add(toDto(branch, "RETAINED", persisted.get(branch), null)); + } + } + return result; + } + + private RagBranchIndexStatusDTO toDto( + String branch, + String role, + RagBranchIndex index, + RagIndexStatus legacyPrimaryStatus) { + if (index == null) { + return legacyPrimaryStatus == null + ? new RagBranchIndexStatusDTO(branch, role, "NOT_INDEXED", null, null, + null, null, null, null) + : legacyPrimaryDto(branch, role, legacyPrimaryStatus); + } + + var generation = index.getActiveGeneration(); + String status = switch (index.getLifecycleStatus()) { + case PENDING -> "PENDING"; + case BUILDING -> "BUILDING"; + case FAILED -> "FAILED"; + case READY -> generation == null ? "NOT_INDEXED" : "READY"; + }; + return new RagBranchIndexStatusDTO( + branch, + role, + status, + generation != null ? generation.getRevision() : index.getCommitHash(), + index.getDesiredCommitHash(), + generation != null ? generation.getFileCount() : null, + generation != null ? generation.getChunkCount() : index.getChunkCount(), + generation != null && generation.getActivatedAt() != null + ? generation.getActivatedAt() : index.getUpdatedAt(), + index.getErrorMessage()); + } + + private RagBranchIndexStatusDTO legacyPrimaryDto( + String branch, + String role, + RagIndexStatus status) { + String displayStatus = switch (status.getStatus()) { + case INDEXING -> "BUILDING"; + case UPDATING -> "BUILDING"; + case INDEXED -> "READY"; + case FAILED -> "FAILED"; + default -> "NOT_INDEXED"; + }; + OffsetDateTime updated = status.getLastIndexedAt() != null + ? status.getLastIndexedAt() : status.getUpdatedAt(); + return new RagBranchIndexStatusDTO( + branch, role, displayStatus, status.getIndexedCommitHash(), null, + status.getTotalFilesIndexed(), status.getChunkCount(), updated, + status.getErrorMessage()); + } + + private String resolvePrimary(Project project, String configuredBranch) { + if (configuredBranch != null && !configuredBranch.isBlank()) { + return configuredBranch.trim(); + } + if (project.getConfiguration().defaultBranch() != null + && !project.getConfiguration().defaultBranch().isBlank()) { + return project.getConfiguration().defaultBranch().trim(); + } + if (project.getDefaultBranch() != null && project.getDefaultBranch().getBranchName() != null) { + return project.getDefaultBranch().getBranchName().trim(); + } + return null; + } +} diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagIndexingTriggerService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagIndexingTriggerService.java index 07ff7bd2c..0608e18df 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagIndexingTriggerService.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/RagIndexingTriggerService.java @@ -117,6 +117,7 @@ public void triggerIndexing( Long projectId, Long userId, String branch, + boolean allConfiguredBranches, SseEmitter emitter ) { try { @@ -148,7 +149,7 @@ public void triggerIndexing( String projectJwt = generateShortLivedProjectJwt(projectId, userId); // Start indexing via pipeline-agent - proxyToPipelineAgent(projectJwt, branch, emitter); + proxyToPipelineAgent(projectJwt, branch, allConfiguredBranches, emitter); } catch (NoSuchElementException e) { sendError(emitter, e.getMessage()); @@ -169,16 +170,24 @@ private String generateShortLivedProjectJwt(Long projectId, Long userId) { ); } - private void proxyToPipelineAgent(String projectJwt, String branch, SseEmitter emitter) { + private void proxyToPipelineAgent( + String projectJwt, + String branch, + boolean allConfiguredBranches, + SseEmitter emitter) { String pipelineUrl = getPipelineAgentBaseUrl(); String indexUrl = pipelineUrl + "/api/rag/index"; Response response = null; try { // Build request body - Map requestBody = branch != null && !branch.isBlank() - ? Map.of("branch", branch) - : Map.of(); + Map requestBody = new java.util.LinkedHashMap<>(); + if (branch != null && !branch.isBlank()) { + requestBody.put("branch", branch.trim()); + } + if (allConfiguredBranches) { + requestBody.put("allConfiguredBranches", true); + } RequestBody body = RequestBody.create( objectMapper.writeValueAsString(requestBody), diff --git a/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusServiceTest.java b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusServiceTest.java new file mode 100644 index 000000000..0a07fc4d7 --- /dev/null +++ b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/RagBranchIndexStatusServiceTest.java @@ -0,0 +1,56 @@ +package org.rostilos.codecrow.webserver.project.service; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexLifecycleStatus; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class RagBranchIndexStatusServiceTest { + + @Test + void reportsPrimaryAndEachExplicitRetainedBranchWithoutTransientIndexes() { + RagBranchIndexRepository branches = mock(RagBranchIndexRepository.class); + RagIndexStatusService projectStatus = mock(RagIndexStatusService.class); + RagBranchIndexStatusService service = new RagBranchIndexStatusService(branches, projectStatus); + + Project project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + ProjectConfig config = new ProjectConfig(); + config.setRagConfig(new RagConfig( + true, "master", null, null, true, 30, List.of("develop"), true)); + project.setConfiguration(config); + + RagBranchIndex develop = new RagBranchIndex(project, "develop", RagBranchIndexKind.DURABLE); + develop.setLifecycleStatus(RagBranchIndexLifecycleStatus.FAILED); + develop.setErrorMessage("archive unavailable"); + develop.setUpdatedAt(OffsetDateTime.parse("2026-08-07T00:00:00Z")); + + RagBranchIndex temporary = new RagBranchIndex(project, "release/candidate", RagBranchIndexKind.TRANSIENT); + temporary.setLifecycleStatus(RagBranchIndexLifecycleStatus.READY); + + when(branches.findByProjectId(42L)).thenReturn(List.of(develop, temporary)); + when(projectStatus.getIndexStatus(project)).thenReturn(Optional.empty()); + + var statuses = service.getConfiguredBranches(project); + + assertThat(statuses).extracting(value -> value.branchName()) + .containsExactly("master", "develop"); + assertThat(statuses.get(0).status()).isEqualTo("NOT_INDEXED"); + assertThat(statuses.get(1)) + .extracting(value -> value.role(), value -> value.status(), value -> value.errorMessage()) + .containsExactly("RETAINED", "FAILED", "archive unavailable"); + } +} diff --git a/python-ecosystem/inference-orchestrator/src/model/dtos.py b/python-ecosystem/inference-orchestrator/src/model/dtos.py index 78321e346..08a957a91 100644 --- a/python-ecosystem/inference-orchestrator/src/model/dtos.py +++ b/python-ecosystem/inference-orchestrator/src/model/dtos.py @@ -104,6 +104,46 @@ class ReviewRequestDto(BaseModel): previousCommitHash: Optional[str] = Field(default=None, description="Previously analyzed commit hash") currentCommitHash: Optional[str] = Field(default=None, description="Current commit hash being analyzed") baseCommitHash: Optional[str] = Field(default=None, description="Immutable pull-request base commit hash") + ragCollectionTarget: Optional[str] = Field( + default=None, + exclude=True, + description="Internal opaque collection target for the selected branch generation", + ) + ragBaseGenerationManifestSha256: Optional[str] = Field( + default=None, + exclude=True, + description="Internal sealed target-generation receipt returned by RAG indexing", + ) + ragPrGenerationFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal exact PR-overlay generation receipt returned by RAG indexing", + ) + ragPrOverlayGenerationManifestSha256: Optional[str] = Field( + default=None, + exclude=True, + description="Internal content-addressed PR-overlay membership seal returned by RAG indexing", + ) + ragBasePluginFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal plugin selection identity of the sealed target generation", + ) + ragBasePluginDescriptorFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal plugin descriptor identity of the sealed target generation", + ) + ragBasePluginImplementationFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal plugin implementation identity of the sealed target generation", + ) + ragBaseIndexRepresentationFingerprint: Optional[str] = Field( + default=None, + exclude=True, + description="Internal index representation identity of the sealed target generation", + ) # File enrichment data (full file contents + pre-computed dependency graph) enrichmentData: Optional[PrEnrichmentDataDto] = Field(default=None, description="Pre-computed file contents and dependency relationships from Java") projectCapabilities: Optional[ProjectCapabilitiesDto] = Field( diff --git a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py index 169379295..49895c8c9 100644 --- a/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py +++ b/python-ecosystem/inference-orchestrator/src/service/rag/rag_client.py @@ -15,6 +15,10 @@ RAG_DEFAULT_TOP_K = int(os.environ.get("RAG_DEFAULT_TOP_K", "15")) +class RagRetrievalError(RuntimeError): + """A required revision-bound RAG retrieval could not be completed.""" + + def _env_int(name: str, default: int) -> int: value = os.environ.get(name) if value is None or not value.strip(): @@ -119,7 +123,13 @@ async def get_pr_context( base_branch: Optional[str] = None, deleted_files: Optional[List[str]] = None, pr_number: Optional[int] = None, - all_pr_changed_files: Optional[List[str]] = None + all_pr_changed_files: Optional[List[str]] = None, + source_revision: Optional[str] = None, + base_revision: Optional[str] = None, + base_generation_manifest_sha256: Optional[str] = None, + pr_generation_fingerprint: Optional[str] = None, + pr_overlay_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict[str, Any]: """ Get relevant context for PR review with multi-branch support. @@ -185,6 +195,24 @@ async def get_pr_context( payload["pr_number"] = pr_number if all_pr_changed_files: payload["all_pr_changed_files"] = all_pr_changed_files + if source_revision: + payload["source_revision"] = source_revision + if base_revision: + payload["base_revision"] = base_revision + if base_generation_manifest_sha256: + payload["base_generation_manifest_sha256"] = ( + base_generation_manifest_sha256 + ) + if pr_generation_fingerprint: + payload["pr_generation_fingerprint"] = ( + pr_generation_fingerprint + ) + if pr_overlay_generation_manifest_sha256: + payload["pr_overlay_generation_manifest_sha256"] = ( + pr_overlay_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target client = await self._get_client() response = await client.post( @@ -229,7 +257,10 @@ async def semantic_search( project: str, branch: str, top_k: int = 5, - filter_language: Optional[str] = None + filter_language: Optional[str] = None, + repository_revision: Optional[str] = None, + repository_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict[str, Any]: """ Perform semantic search in the repository. @@ -258,6 +289,14 @@ async def semantic_search( } if filter_language: payload["filter_language"] = filter_language + if repository_revision: + payload["repository_revision"] = repository_revision + if repository_generation_manifest_sha256: + payload["repository_generation_manifest_sha256"] = ( + repository_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target client = await self._get_client() response = await client.post( @@ -314,7 +353,10 @@ async def search_for_duplicates( branch: str, queries: List[str], top_k: int = 8, - base_branch: Optional[str] = None + base_branch: Optional[str] = None, + repository_revision: Optional[str] = None, + repository_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> List[Dict[str, Any]]: """ Perform duplication-oriented semantic search to find existing implementations @@ -336,6 +378,16 @@ async def search_for_duplicates( """ if not self.enabled or not queries: return [] + exact_binding_values = ( + repository_revision, + repository_generation_manifest_sha256, + ) + exact_binding_active = any(exact_binding_values) + if exact_binding_active and not all(exact_binding_values): + raise RagRetrievalError( + "revision-bound duplication search requires both repository " + "revision and generation receipt" + ) max_queries = max(1, _env_int("REVIEW_DUPLICATION_RAG_MAX_QUERIES", 8)) query_timeout = max( @@ -368,6 +420,14 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: } if base_branch: payload["base_branch"] = base_branch + if repository_revision: + payload["repository_revision"] = repository_revision + if repository_generation_manifest_sha256: + payload["repository_generation_manifest_sha256"] = ( + repository_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target async with semaphore: started_at = datetime.now() @@ -383,12 +443,20 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: result = response.json() except asyncio.TimeoutError: elapsed_ms = (datetime.now() - started_at).total_seconds() * 1000 - logger.debug( - "Duplication search query timed out after %.0fms", - elapsed_ms, + message = ( + "revision-bound duplication search query timed out " + f"after {elapsed_ms:.0f}ms" ) + if exact_binding_active: + raise RagRetrievalError(message) + logger.debug(message) return [] except Exception as e: + if exact_binding_active: + raise RagRetrievalError( + "revision-bound duplication search query failed: " + f"{type(e).__name__}: {e}" + ) from e logger.debug(f"Duplication search query failed: {e}") return [] @@ -401,7 +469,7 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: result_groups = await asyncio.gather( *(_run_query(query_text) for query_text in selected_queries), - return_exceptions=True, + return_exceptions=not exact_binding_active, ) all_results: List[Dict[str, Any]] = [] for group in result_groups: @@ -421,7 +489,14 @@ async def _run_query(query_text: str) -> List[Dict[str, Any]]: ) return all_results + except RagRetrievalError: + raise except Exception as e: + if exact_binding_active: + raise RagRetrievalError( + "revision-bound duplication search failed: " + f"{type(e).__name__}: {e}" + ) from e logger.warning(f"Failed duplication search: {e}") return [] @@ -434,7 +509,13 @@ async def get_deterministic_context( limit_per_file: int = 10, pr_number: Optional[int] = None, pr_changed_files: Optional[List[str]] = None, - additional_identifiers: Optional[List[str]] = None + additional_identifiers: Optional[List[str]] = None, + source_revision: Optional[str] = None, + base_revision: Optional[str] = None, + base_generation_manifest_sha256: Optional[str] = None, + pr_generation_fingerprint: Optional[str] = None, + pr_overlay_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict[str, Any]: """ Get context using DETERMINISTIC metadata-based retrieval. @@ -481,6 +562,24 @@ async def get_deterministic_context( payload["pr_changed_files"] = pr_changed_files if additional_identifiers: payload["additional_identifiers"] = additional_identifiers + if source_revision: + payload["source_revision"] = source_revision + if base_revision: + payload["base_revision"] = base_revision + if base_generation_manifest_sha256: + payload["base_generation_manifest_sha256"] = ( + base_generation_manifest_sha256 + ) + if pr_generation_fingerprint: + payload["pr_generation_fingerprint"] = ( + pr_generation_fingerprint + ) + if pr_overlay_generation_manifest_sha256: + payload["pr_overlay_generation_manifest_sha256"] = ( + pr_overlay_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target client = await self._get_client() response = await client.post( @@ -537,6 +636,8 @@ async def index_pr_files( plugin_detection_evidence: Optional[Dict[str, List[str]]] = None, plugin_fingerprint: str = "sha256:" + "0" * 64, plugin_descriptor_fingerprint: str = "sha256:" + "0" * 64, + base_generation_manifest_sha256: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict[str, Any]: """ Index PR files into the main collection with PR-specific metadata. @@ -590,6 +691,12 @@ async def index_pr_files( "plugin_descriptor_fingerprint": plugin_descriptor_fingerprint, "files": files } + if base_generation_manifest_sha256: + payload["base_generation_manifest_sha256"] = ( + base_generation_manifest_sha256 + ) + if collection_target: + payload["collection_target"] = collection_target client = await self._get_client() response = await client.post( @@ -631,7 +738,8 @@ async def delete_pr_files( self, workspace: str, project: str, - pr_number: int + pr_number: int, + collection_target: Optional[str] = None, ) -> bool: """ Delete all indexed points for a specific PR. @@ -652,7 +760,11 @@ async def delete_pr_files( try: client = await self._get_client() response = await client.delete( - f"{self.base_url}/index/pr-files/{workspace}/{project}/{pr_number}" + f"{self.base_url}/index/pr-files/{workspace}/{project}/{pr_number}", + params=( + {"collection_target": collection_target} + if collection_target else None + ), ) response.raise_for_status() result = response.json() diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py index f4a35d8e6..bf31881f2 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py @@ -182,6 +182,9 @@ def _emit_review_evidence_completed( review_units: Optional[Stage1ReviewUnitState] = None, rag_state: Optional[Stage1RagState] = None, candidate_ledger: Optional[CandidateEvidenceLedger] = None, + *, + request: ReviewRequestDto, + pr_indexed: bool, ) -> None: """Expose compact host-owned completion evidence without prompt/source data.""" if callback is None: @@ -227,6 +230,43 @@ def _emit_review_evidence_completed( rag_state.exact_evidence_by_id if rag_state is not None else {} ), }, + "revisionBinding": { + "prIndexed": pr_indexed, + "pullRequestId": request.pullRequestId, + "targetBranch": request.targetBranchName, + "sourceRevision": ( + request.currentCommitHash or request.commitHash + ), + "baseRevision": request.baseCommitHash, + "baseGenerationManifestSha256": ( + request.ragBaseGenerationManifestSha256 + if pr_indexed else None + ), + "prGenerationFingerprint": ( + request.ragPrGenerationFingerprint + if pr_indexed else None + ), + "prOverlayGenerationManifestSha256": ( + request.ragPrOverlayGenerationManifestSha256 + if pr_indexed else None + ), + "basePluginFingerprint": ( + request.ragBasePluginFingerprint + if pr_indexed else None + ), + "basePluginDescriptorFingerprint": ( + request.ragBasePluginDescriptorFingerprint + if pr_indexed else None + ), + "basePluginImplementationFingerprint": ( + request.ragBasePluginImplementationFingerprint + if pr_indexed else None + ), + "baseIndexRepresentationFingerprint": ( + request.ragBaseIndexRepresentationFingerprint + if pr_indexed else None + ), + }, }) @@ -386,6 +426,10 @@ async def _index_pr_files( base_branch=identity.target_branch, source_revision=identity.head_revision, base_revision=identity.base_revision, + collection_target=request.ragCollectionTarget, + base_generation_manifest_sha256=( + request.ragBaseGenerationManifestSha256 + ), repository_plugins=( list(capabilities.repositoryPlugins) if capabilities else [] ), @@ -409,6 +453,32 @@ async def _index_pr_files( request, result.get("effective_project_capabilities"), ) + request.ragBaseGenerationManifestSha256 = ( + result.get("base_generation_manifest_sha256") + or request.ragBaseGenerationManifestSha256 + ) + request.ragPrGenerationFingerprint = result.get( + "generation_fingerprint" + ) + request.ragPrOverlayGenerationManifestSha256 = result.get( + "overlay_generation_manifest_sha256" + ) + request.ragBasePluginFingerprint = ( + result.get("plugin_fingerprint") + or request.ragBasePluginFingerprint + ) + request.ragBasePluginDescriptorFingerprint = ( + result.get("plugin_descriptor_fingerprint") + or request.ragBasePluginDescriptorFingerprint + ) + request.ragBasePluginImplementationFingerprint = ( + result.get("plugin_implementation_fingerprint") + or request.ragBasePluginImplementationFingerprint + ) + request.ragBaseIndexRepresentationFingerprint = ( + result.get("index_representation_fingerprint") + or request.ragBaseIndexRepresentationFingerprint + ) self._pr_indexed = True self._repository_review_groups = tuple( tuple( @@ -462,7 +532,8 @@ async def _cleanup_pr_files(self, request: ReviewRequestDto) -> None: await self.rag_client.delete_pr_files( workspace=request.projectWorkspace, project=request.projectNamespace, - pr_number=self._pr_number + pr_number=self._pr_number, + collection_target=request.ragCollectionTarget, ) logger.info(f"Cleaned up PR #{self._pr_number} indexed data") except Exception as e: @@ -921,6 +992,8 @@ async def orchestrate_review( self.event_callback, hunk_coverage, candidate_ledger=candidate_ledger, + request=request, + pr_indexed=self._pr_indexed, ) logger.info( "Review completed locally: every acquired hunk has a " @@ -1379,6 +1452,8 @@ async def orchestrate_review( stage_1_review_unit_state, stage_1_rag_state, candidate_ledger, + request=request, + pr_indexed=self._pr_indexed, ) logger.info("Review hunk coverage complete: %s", hunk_coverage.summary()) diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py index ded1f4e3f..f293e5657 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_1_file_review.py @@ -234,14 +234,19 @@ def _capture_deterministic_retrieval_state( if _rag_response_error(deterministic_response): rag_state.deterministic_retrieval_states.append("failed") return + rag_state.deterministic_retrieval_states.append( + _deterministic_retrieval_state(deterministic_response) + ) + + +def _deterministic_retrieval_state( + deterministic_response: Optional[Dict[str, Any]], +) -> str: context = _unwrap_rag_context(deterministic_response) metadata = context.get("_metadata") if isinstance(context, dict) else None - retrieval_state = ( - str(metadata.get("retrieval_state")) - if isinstance(metadata, dict) and metadata.get("retrieval_state") - else "unknown" - ) - rag_state.deterministic_retrieval_states.append(retrieval_state) + if isinstance(metadata, dict) and metadata.get("retrieval_state"): + return str(metadata["retrieval_state"]).strip().casefold() + return "unknown" def _rag_response_error(response: Optional[Dict[str, Any]]) -> Optional[str]: @@ -1212,6 +1217,25 @@ async def create_smart_batches_wrapper( "using local/enrichment grouping without a repository RAG lookup" ) batching_rag_client = None + exact_receipt_values = tuple( + value + for value in ( + getattr(request, "ragBaseGenerationManifestSha256", None), + getattr(request, "ragPrGenerationFingerprint", None), + getattr( + request, + "ragPrOverlayGenerationManifestSha256", + None, + ), + ) + if isinstance(value, str) and value + ) + if any(exact_receipt_values): + logger.info( + "Stage 1 smart-batching RAG discovery disabled while exact " + "base/overlay generation receipts are active" + ) + batching_rag_client = None enrichment_data = getattr(request, 'enrichmentData', None) @@ -1534,6 +1558,20 @@ def _expand_oversized_diff_batches( # ── RAG Context ─────────────────────────────────────────────── +def _is_exact_revision_bound( + request: ReviewRequestDto, + pr_indexed: bool, +) -> bool: + return bool( + pr_indexed + and (request.currentCommitHash or request.commitHash) + and request.baseCommitHash + and request.ragBaseGenerationManifestSha256 + and request.ragPrGenerationFingerprint + and request.ragPrOverlayGenerationManifestSha256 + ) + + async def fetch_batch_rag_context( rag_client, request: ReviewRequestDto, @@ -1547,9 +1585,16 @@ async def fetch_batch_rag_context( batch_raw_diffs: Optional[List[str]] = None, rag_state: Optional[Stage1RagState] = None, ) -> Optional[Dict[str, Any]]: + exact_revision_bound = _is_exact_revision_bound(request, pr_indexed) if not rag_client: + if exact_revision_bound: + raise RuntimeError( + "revision-bound Stage 1 retrieval requires a RAG client" + ) return None + duplication_task: Optional[asyncio.Task] = None + try: rag_branch = request.get_rag_branch() base_branch = request.get_rag_base_branch() @@ -1560,6 +1605,8 @@ async def fetch_batch_rag_context( {"status": "error", "error": message}, rag_state, ) + if exact_revision_bound: + raise RuntimeError(message) return None # Scale top_k based on batch priority to ensure adequate context @@ -1571,6 +1618,27 @@ async def fetch_batch_rag_context( pr_number = request.pullRequestId if pr_indexed else None all_pr_files = request.changedFiles if pr_indexed else None + source_revision = ( + request.currentCommitHash or request.commitHash + if pr_indexed + else None + ) + base_revision = request.baseCommitHash if pr_indexed else None + base_generation_receipt = ( + request.ragBaseGenerationManifestSha256 + if pr_indexed + else None + ) + pr_generation_fingerprint = ( + request.ragPrGenerationFingerprint + if pr_indexed + else None + ) + pr_overlay_generation_manifest_sha256 = ( + request.ragPrOverlayGenerationManifestSha256 + if pr_indexed + else None + ) context = None @@ -1579,12 +1647,26 @@ async def _fetch_deterministic_context() -> Optional[Dict[str, Any]]: return await rag_client.get_deterministic_context( workspace=request.projectWorkspace, project=request.projectNamespace, - branches=[branch for branch in [rag_branch, base_branch] if branch], + branches=list(dict.fromkeys( + branch + for branch in (rag_branch, base_branch) + if branch + )), file_paths=batch_file_paths, limit_per_file=5, pr_number=pr_number, pr_changed_files=all_pr_files, additional_identifiers=enrichment_identifiers, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_receipt + ), + pr_generation_fingerprint=pr_generation_fingerprint, + pr_overlay_generation_manifest_sha256=( + pr_overlay_generation_manifest_sha256 + ), + collection_target=request.ragCollectionTarget, ) except Exception as det_err: logger.warning("Deterministic RAG lookup failed: %s", det_err) @@ -1613,6 +1695,14 @@ async def _fetch_semantic_context() -> Optional[Dict[str, Any]]: pr_number=pr_number, all_pr_changed_files=all_pr_files, deleted_files=request.deletedFiles or None, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_receipt, + pr_generation_fingerprint=pr_generation_fingerprint, + pr_overlay_generation_manifest_sha256=( + pr_overlay_generation_manifest_sha256 + ), + collection_target=request.ragCollectionTarget, ) async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: @@ -1649,8 +1739,15 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: queries=duplication_queries, top_k=8, base_branch=base_branch, + repository_revision=base_revision, + repository_generation_manifest_sha256=( + base_generation_receipt + ), + collection_target=request.ragCollectionTarget, ) except Exception as dup_err: + if base_revision or base_generation_receipt: + raise logger.debug(f"Duplication search skipped: {dup_err}") return None @@ -1670,6 +1767,22 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: deterministic_response, rag_state, ) + if deterministic_error and exact_revision_bound: + raise RuntimeError( + "revision-bound deterministic RAG retrieval failed: " + f"{deterministic_error}" + ) + deterministic_retrieval_state = _deterministic_retrieval_state( + deterministic_response + ) + if ( + exact_revision_bound + and deterministic_retrieval_state != "complete" + ): + raise RuntimeError( + "revision-bound deterministic RAG retrieval is not complete: " + f"{deterministic_retrieval_state}" + ) if deterministic_chunks: context = {"relevant_code": deterministic_chunks} logger.info( @@ -1682,9 +1795,22 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: semantic_fill = max(0, top_k - det_count) rag_response = None - if semantic_fill > 0 and rag_state and rag_state.semantic_disabled: + semantic_fill_enabled = ( + semantic_fill > 0 and SEMANTIC_RAG_FILLER_ENABLED + ) + if semantic_fill > 0 and not SEMANTIC_RAG_FILLER_ENABLED: + logger.info( + "Semantic RAG filler skipped by " + "REVIEW_SEMANTIC_RAG_FILLER_ENABLED" + ) + elif semantic_fill_enabled and rag_state and rag_state.semantic_disabled: logger.info("Semantic RAG filler skipped: %s", rag_state.semantic_disable_reason) - elif semantic_fill > 0: + if exact_revision_bound: + raise RuntimeError( + "revision-bound semantic RAG retrieval is disabled after " + f"a prior failure: {rag_state.semantic_disable_reason}" + ) + elif semantic_fill_enabled: try: rag_response = await asyncio.wait_for( _fetch_semantic_context(), @@ -1705,6 +1831,10 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: "Semantic RAG filler timed out after %ss; disabling for remaining Stage 1 batches", SEMANTIC_RAG_TIMEOUT_SECONDS, ) + if exact_revision_bound: + raise RuntimeError( + "revision-bound semantic RAG retrieval timed out" + ) except Exception as sem_err: rag_response = None if rag_state: @@ -1712,6 +1842,17 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: rag_state.semantic_disabled = True rag_state.semantic_disable_reason = str(sem_err) logger.warning("Semantic RAG filler failed; disabling for remaining Stage 1 batches: %s", sem_err) + if exact_revision_bound: + raise + + if ( + semantic_fill_enabled + and exact_revision_bound + and rag_response is None + ): + raise RuntimeError( + "revision-bound semantic RAG retrieval returned no response" + ) if semantic_fill > 0 and rag_response: sem_context = _unwrap_rag_context(rag_response) @@ -1811,7 +1952,13 @@ async def _fetch_duplication_context() -> Optional[List[Dict[str, Any]]]: return None except Exception as e: + if duplication_task is not None and not duplication_task.done(): + duplication_task.cancel() + if duplication_task is not None: + await asyncio.gather(duplication_task, return_exceptions=True) logger.warning(f"Failed to fetch per-batch RAG context: {e}") + if exact_revision_bound: + raise return None @@ -2254,7 +2401,7 @@ async def review_file_batch( str, tuple[Dict[str, Any], ...] ] = {} - if rag_client: + if rag_client or _is_exact_revision_bound(request, pr_indexed): batch_rag_context = await fetch_batch_rag_context( rag_client, request, batch_file_paths, batch_diff_snippets, pr_indexed, llm_reranker=llm_reranker, diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py index 47172ab14..880cb3e73 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_2_cross_file.py @@ -686,6 +686,37 @@ async def _fetch_cross_module_context( if not rag_client: return "" + base_revision_value = getattr(request, "baseCommitHash", None) + base_generation_receipt_value = getattr( + request, + "ragBaseGenerationManifestSha256", + None, + ) + base_revision = ( + base_revision_value + if isinstance(base_revision_value, str) and base_revision_value + else None + ) + base_generation_receipt = ( + base_generation_receipt_value + if ( + isinstance(base_generation_receipt_value, str) + and base_generation_receipt_value + ) + else None + ) + if not base_revision and not base_generation_receipt: + logger.info( + "Stage 2 cross-module RAG skipped: no exact target-generation lease" + ) + return "" + if not base_revision or not base_generation_receipt: + logger.warning( + "Stage 2 cross-module RAG requires both immutable target revision " + "and generation receipt" + ) + return "" + try: rag_branch = request.get_rag_branch() base_branch = request.get_rag_base_branch() @@ -733,6 +764,11 @@ async def _fetch_cross_module_context( queries=unique_queries, top_k=6, base_branch=base_branch, + repository_revision=base_revision, + repository_generation_manifest_sha256=( + base_generation_receipt + ), + collection_target=getattr(request, "ragCollectionTarget", None), ) if not dup_results: @@ -752,5 +788,9 @@ async def _fetch_cross_module_context( return formatted except Exception as e: - logger.warning(f"Failed to fetch cross-module context for Stage 2: {e}") + logger.warning( + "Revision-bound cross-module context unavailable for Stage 2: %s: %s", + type(e).__name__, + e, + ) return "" diff --git a/python-ecosystem/inference-orchestrator/src/service/review/prompt_dry_run.py b/python-ecosystem/inference-orchestrator/src/service/review/prompt_dry_run.py index e749f0a51..229e7bdb4 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/prompt_dry_run.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/prompt_dry_run.py @@ -713,6 +713,14 @@ async def search_for_duplicates(self, **_: Any) -> list[dict[str, Any]]: async def index_pr_files(self, **_: Any) -> dict[str, Any]: # Report the shape expected by the orchestrator so it follows the same # post-index prompt path, while keeping the operation entirely local. + # When deterministic retrieval is intentionally absent, do not mint + # synthetic exact-generation receipts: Stage 1 would correctly require + # a complete exact retrieval for receipts that claim such a generation. + if not self._enabled: + return { + "status": "skipped", + "reason": "deterministic retrieval disabled for prompt dry run", + } effective = None if self._project_capabilities is not None: from service.review.plugin_context import _plugin_host @@ -733,6 +741,13 @@ async def index_pr_files(self, **_: Any) -> dict[str, Any]: return { "status": "indexed", "chunks_indexed": 0, + "base_generation_manifest_sha256": "0" * 64, + "generation_fingerprint": "sha256:" + "0" * 64, + "overlay_generation_manifest_sha256": "0" * 64, + "plugin_fingerprint": "sha256:" + "0" * 64, + "plugin_descriptor_fingerprint": "sha256:" + "0" * 64, + "plugin_implementation_fingerprint": "sha256:" + "0" * 64, + "index_representation_fingerprint": "sha256:" + "0" * 64, "effective_project_capabilities": effective, } @@ -795,6 +810,8 @@ def capture_event(event: dict[str, Any]) -> None: include_deterministic_rag, safe_request.projectCapabilities, ) + if include_deterministic_rag + else None ) orchestrator = MultiStageReviewOrchestrator( llm=llm, @@ -836,7 +853,11 @@ def capture_event(event: dict[str, Any]) -> None: provider=request.aiProvider, model=request.aiModel, deterministic_rag_requests=( - None if full_pipeline_context else dry_rag.deterministic_requests + None + if full_pipeline_context + else dry_rag.deterministic_requests + if dry_rag is not None + else 0 ), deterministic_rag_enabled=( full_pipeline_rag_enabled diff --git a/python-ecosystem/inference-orchestrator/src/service/review/quality_capture.py b/python-ecosystem/inference-orchestrator/src/service/review/quality_capture.py index 560967aeb..afd48752d 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/quality_capture.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/quality_capture.py @@ -28,6 +28,9 @@ logger = logging.getLogger(__name__) _SAFE_FILENAME = re.compile(r"[^A-Za-z0-9_.-]+") +_IMMUTABLE_GIT_REVISION = re.compile(r"(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})") +_SHA256_HEX = re.compile(r"[0-9a-f]{64}") +_SHA256_FINGERPRINT = re.compile(r"sha256:[0-9a-f]{64}") _SECRET_KEYS = { "access_token", "accesstoken", @@ -355,6 +358,43 @@ def _selected_project_ids() -> set[int]: return selected +def _provider_reported_models(value: Any) -> list[str]: + """Return model IDs explicitly reported inside a provider callback result.""" + + observed: set[str] = set() + + def inspect_metadata(current: Any) -> None: + if not isinstance(current, dict): + return + for key, child in current.items(): + normalized = str(key).strip().casefold().replace("-", "_") + if normalized in {"model", "model_id", "model_name"}: + if isinstance(child, str) and child.strip(): + observed.add(child.strip()) + elif normalized in { + "llm_output", + "metadata", + "response_metadata", + }: + inspect_metadata(child) + + if isinstance(value, dict): + inspect_metadata(value.get("llm_output")) + generations = value.get("generations") + if isinstance(generations, list): + for group in generations: + if not isinstance(group, list): + continue + for generation in group: + if not isinstance(generation, dict): + continue + inspect_metadata(generation.get("generation_info")) + message = generation.get("message") + if isinstance(message, dict): + inspect_metadata(message.get("response_metadata")) + return sorted(observed) + + class _ProviderBoundaryCallback: """Collect the underlying provider result before structured parsing.""" @@ -380,9 +420,11 @@ def on_llm_new_token(self, *_: Any, **__: Any) -> None: return None def on_llm_end(self, response: Any, **_: Any) -> None: + safe_response = _json_safe(response) self.events.append({ "status": "completed", - "response": _json_safe(response), + "providerReportedModels": _provider_reported_models(safe_response), + "response": safe_response, }) def on_llm_error(self, error: BaseException, **_: Any) -> None: @@ -827,6 +869,150 @@ def _terminal_pipeline_evidence(event: Any) -> Optional[dict[str, Any]]: "terminal pipeline evidence has invalid retrieval.semanticDisabled" ) + revision_binding = event.get("revisionBinding") + if not isinstance(revision_binding, dict): + raise ValueError("terminal pipeline evidence has no revision binding") + pr_indexed = revision_binding.get("prIndexed") + pull_request_id = revision_binding.get("pullRequestId") + target_branch = revision_binding.get("targetBranch") + source_revision = revision_binding.get("sourceRevision") + base_revision = revision_binding.get("baseRevision") + base_manifest = revision_binding.get("baseGenerationManifestSha256") + pr_fingerprint = revision_binding.get("prGenerationFingerprint") + overlay_manifest = revision_binding.get( + "prOverlayGenerationManifestSha256" + ) + base_plugin_fingerprint = revision_binding.get( + "basePluginFingerprint" + ) + base_plugin_descriptor_fingerprint = revision_binding.get( + "basePluginDescriptorFingerprint" + ) + base_plugin_implementation_fingerprint = revision_binding.get( + "basePluginImplementationFingerprint" + ) + base_index_representation_fingerprint = revision_binding.get( + "baseIndexRepresentationFingerprint" + ) + if not isinstance(pr_indexed, bool): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.prIndexed" + ) + if ( + pull_request_id is not None + and ( + not isinstance(pull_request_id, int) + or isinstance(pull_request_id, bool) + or pull_request_id < 1 + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.pullRequestId" + ) + if not isinstance(target_branch, str) or not target_branch.strip(): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.targetBranch" + ) + if ( + not isinstance(source_revision, str) + or _IMMUTABLE_GIT_REVISION.fullmatch(source_revision) is None + ): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.sourceRevision" + ) + if ( + base_revision is not None + and ( + not isinstance(base_revision, str) + or _IMMUTABLE_GIT_REVISION.fullmatch(base_revision) is None + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid revisionBinding.baseRevision" + ) + if ( + base_manifest is not None + and ( + not isinstance(base_manifest, str) + or _SHA256_HEX.fullmatch(base_manifest) is None + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid " + "revisionBinding.baseGenerationManifestSha256" + ) + if ( + pr_fingerprint is not None + and ( + not isinstance(pr_fingerprint, str) + or _SHA256_FINGERPRINT.fullmatch(pr_fingerprint) is None + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid " + "revisionBinding.prGenerationFingerprint" + ) + if ( + overlay_manifest is not None + and ( + not isinstance(overlay_manifest, str) + or _SHA256_HEX.fullmatch(overlay_manifest) is None + ) + ): + raise ValueError( + "terminal pipeline evidence has invalid " + "revisionBinding.prOverlayGenerationManifestSha256" + ) + for field, value in ( + ("basePluginFingerprint", base_plugin_fingerprint), + ( + "basePluginDescriptorFingerprint", + base_plugin_descriptor_fingerprint, + ), + ( + "basePluginImplementationFingerprint", + base_plugin_implementation_fingerprint, + ), + ( + "baseIndexRepresentationFingerprint", + base_index_representation_fingerprint, + ), + ): + if value is not None and ( + not isinstance(value, str) + or _SHA256_FINGERPRINT.fullmatch(value) is None + ): + raise ValueError( + "terminal pipeline evidence has invalid " + f"revisionBinding.{field}" + ) + if pr_indexed and ( + pull_request_id is None + or base_revision is None + or base_manifest is None + or pr_fingerprint is None + or overlay_manifest is None + or base_plugin_fingerprint is None + or base_plugin_descriptor_fingerprint is None + or base_plugin_implementation_fingerprint is None + or base_index_representation_fingerprint is None + ): + raise ValueError( + "terminal pipeline evidence has incomplete indexed revision binding" + ) + if not pr_indexed and ( + base_manifest is not None + or pr_fingerprint is not None + or overlay_manifest is not None + or base_plugin_fingerprint is not None + or base_plugin_descriptor_fingerprint is not None + or base_plugin_implementation_fingerprint is not None + or base_index_representation_fingerprint is not None + ): + raise ValueError( + "terminal pipeline evidence has receipts for an unindexed PR overlay" + ) + return { "state": "review_evidence_completed", "hunkCoverage": normalized_hunks, @@ -843,6 +1029,26 @@ def _terminal_pipeline_evidence(event: Any) -> Optional[dict[str, Any]]: "semanticDisabled": semantic_disabled, "exactEvidenceIds": exact_evidence_ids, }, + "revisionBinding": { + "prIndexed": pr_indexed, + "pullRequestId": pull_request_id, + "targetBranch": target_branch, + "sourceRevision": source_revision, + "baseRevision": base_revision, + "baseGenerationManifestSha256": base_manifest, + "prGenerationFingerprint": pr_fingerprint, + "prOverlayGenerationManifestSha256": overlay_manifest, + "basePluginFingerprint": base_plugin_fingerprint, + "basePluginDescriptorFingerprint": ( + base_plugin_descriptor_fingerprint + ), + "basePluginImplementationFingerprint": ( + base_plugin_implementation_fingerprint + ), + "baseIndexRepresentationFingerprint": ( + base_index_representation_fingerprint + ), + }, } @@ -1022,6 +1228,55 @@ def _prune(self) -> None: for stale in terminal[self._max_files:]: stale.unlink(missing_ok=True) + def receipt(self) -> dict[str, Any]: + """Return a source-free receipt for the completed capture artifact.""" + + if ( + self._artifact.get("status") not in {"completed", "failed"} + or not self._artifact.get("captureDigest") + ): + raise ValueError("quality capture receipt requires a terminal artifact") + call_receipts = [] + all_reported_models: set[str] = set() + model_evidence_complete = True + for call in self._artifact["calls"]: + reported = sorted({ + model + for event in call.get("providerEvents") or [] + if isinstance(event, dict) + for model in event.get("providerReportedModels") or [] + if isinstance(model, str) and model + }) + all_reported_models.update(reported) + if call.get("status") == "completed" and not reported: + model_evidence_complete = False + call_receipts.append({ + "sequence": call.get("sequence"), + "stage": call.get("stage"), + "status": call.get("status"), + "providerCallCount": call.get("providerCallCount"), + "providerReportedModels": reported, + "promptDigest": call.get("promptDigest"), + "responseDigest": call.get("responseDigest"), + }) + receipt = { + "kind": "review-quality-capture-receipt", + "status": self._artifact["status"], + "artifactContainerPath": self.container_path, + "captureDigest": self._artifact["captureDigest"], + "provider": self._artifact["provider"], + "requestedModel": self._artifact["model"], + "providerReportedModels": sorted(all_reported_models), + "providerModelEvidenceComplete": model_evidence_complete, + "modelBoundaryInvocations": self._artifact[ + "modelBoundaryInvocations" + ], + "providerCalls": self._artifact["providerCalls"], + "calls": call_receipts, + } + receipt["receiptDigest"] = _digest(receipt) + return receipt + async def invoke( self, delegate: Any, @@ -1280,10 +1535,16 @@ def with_structured_output( include_raw=include_raw, **kwargs, ) + bindings = dict(self._bindings) + bindings["structured_output"] = { + "include_raw": include_raw, + "options": dict(kwargs), + } return self._clone( delegate, schema=schema, include_raw=include_raw, + bindings=bindings, ) def bind_tools( @@ -1293,8 +1554,13 @@ def bind_tools( ) -> "ReviewQualityCaptureLLM": materialized = tuple(tools) delegate = self._delegate.bind_tools(materialized, **kwargs) + bindings = dict(self._bindings) + bindings["tool_binding"] = { + "options": dict(kwargs), + } return self._clone( delegate, + bindings=bindings, tools=tuple(_tool_descriptor(tool) for tool in materialized), ) diff --git a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py index 338dab1d5..b82f7d644 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py @@ -33,6 +33,16 @@ logger = logging.getLogger(__name__) + +def allow_unbound_global_rag_fallback(request: ReviewRequestDto) -> bool: + """Legacy fallback is unsafe when the review carries an exact PR lease.""" + is_exact_pr = bool( + request.pullRequestId + and request.baseCommitHash + and (request.currentCommitHash or request.commitHash) + ) + return not is_exact_pr + class ReviewService: """Service class for handling code review requests with streaming support.""" @@ -103,6 +113,12 @@ async def process_review_request( response, failed=review_response_indicates_failure(response), ) + self._emit_event(review_event_callback, { + "type": "status", + "state": "review_quality_capture_completed", + "message": "Review quality capture completed", + "qualityCapture": quality_capture.receipt(), + }) return response async def _process_prompt_dry_run( @@ -383,7 +399,11 @@ async def _process_review( # task is only awaited if a batch cannot obtain per-batch # context. Branch reconciliation does not need it. request_rag_client = self._rag_client_for_request(request) - if needs_multistage_review and request_rag_client is not None: + if ( + needs_multistage_review + and request_rag_client is not None + and allow_unbound_global_rag_fallback(request) + ): rag_context_task = asyncio.create_task( self._fetch_rag_context( request, diff --git a/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py b/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py index 8fd7d50fa..82352e3fc 100644 --- a/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py +++ b/python-ecosystem/inference-orchestrator/tests/prompt_dry_run_neutral_fixture.py @@ -9,6 +9,13 @@ SECRET_API_KEY = "dry-run-provider-key-must-never-be-used-or-returned" HEAD_REVISION = "1" * 40 BASE_REVISION = "2" * 40 +BASE_GENERATION_MANIFEST = "3" * 64 +PR_GENERATION_FINGERPRINT = "sha256:" + "4" * 64 +PR_OVERLAY_GENERATION_MANIFEST = "5" * 64 +BASE_PLUGIN_FINGERPRINT = "sha256:" + "6" * 64 +BASE_PLUGIN_DESCRIPTOR_FINGERPRINT = "sha256:" + "7" * 64 +BASE_PLUGIN_IMPLEMENTATION_FINGERPRINT = "sha256:" + "8" * 64 +BASE_INDEX_REPRESENTATION_FINGERPRINT = "sha256:" + "9" * 64 class DeterministicRagSpy: @@ -42,7 +49,25 @@ async def search_for_duplicates(self, **_kwargs): async def index_pr_files(self, **kwargs): self.index_requests.append(kwargs) - return {"status": "indexed", "chunks_indexed": 0} + return { + "status": "indexed", + "chunks_indexed": 0, + "base_generation_manifest_sha256": BASE_GENERATION_MANIFEST, + "generation_fingerprint": PR_GENERATION_FINGERPRINT, + "overlay_generation_manifest_sha256": ( + PR_OVERLAY_GENERATION_MANIFEST + ), + "plugin_fingerprint": BASE_PLUGIN_FINGERPRINT, + "plugin_descriptor_fingerprint": ( + BASE_PLUGIN_DESCRIPTOR_FINGERPRINT + ), + "plugin_implementation_fingerprint": ( + BASE_PLUGIN_IMPLEMENTATION_FINGERPRINT + ), + "index_representation_fingerprint": ( + BASE_INDEX_REPRESENTATION_FINGERPRINT + ), + } async def delete_pr_files(self, **kwargs): self.delete_requests.append(kwargs) diff --git a/python-ecosystem/inference-orchestrator/tests/test_candidate_ledger.py b/python-ecosystem/inference-orchestrator/tests/test_candidate_ledger.py index a9818bcfa..c3c30208d 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_candidate_ledger.py +++ b/python-ecosystem/inference-orchestrator/tests/test_candidate_ledger.py @@ -264,6 +264,20 @@ def test_terminal_capture_accepts_deterministic_candidate_ledger(): "semanticDisabled": False, "exactEvidenceIds": 0, }, + "revisionBinding": { + "prIndexed": True, + "pullRequestId": 12, + "targetBranch": "main", + "sourceRevision": "a" * 40, + "baseRevision": "b" * 40, + "baseGenerationManifestSha256": "c" * 64, + "prGenerationFingerprint": "sha256:" + "d" * 64, + "prOverlayGenerationManifestSha256": "e" * 64, + "basePluginFingerprint": "sha256:" + "1" * 64, + "basePluginDescriptorFingerprint": "sha256:" + "2" * 64, + "basePluginImplementationFingerprint": "sha256:" + "3" * 64, + "baseIndexRepresentationFingerprint": "sha256:" + "4" * 64, + }, }) assert evidence["candidates"]["generated"] == 1 diff --git a/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py b/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py index 32bb748ee..b834b212b 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py +++ b/python-ecosystem/inference-orchestrator/tests/test_prompt_dry_run.py @@ -50,10 +50,12 @@ async def test_non_reviewable_hunk_manifest_completes_without_model_stage(): for hunk in generated.hunks ] rag = DeterministicRagSpy() + events = [] orchestrator = MultiStageReviewOrchestrator( llm=object(), mcp_client=None, rag_client=rag, + event_callback=events.append, ) result = await orchestrator.orchestrate_review( @@ -66,6 +68,16 @@ async def test_non_reviewable_hunk_manifest_completes_without_model_stage(): assert rag.index_requests == [] assert rag.requests == [] assert rag.semantic_requests == [] + terminal_event = next( + event + for event in events + if event.get("state") == "review_evidence_completed" + ) + assert terminal_event["revisionBinding"]["prIndexed"] is False + assert ( + terminal_event["revisionBinding"]["baseGenerationManifestSha256"] + is None + ) @pytest.mark.asyncio @@ -796,6 +808,25 @@ async def test_full_pipeline_capture_persists_real_context_artifact( event.get("state") == "review_evidence_completed" for event in forwarded_events ) + terminal_event = next( + event + for event in forwarded_events + if event.get("state") == "review_evidence_completed" + ) + assert terminal_event["revisionBinding"] == { + "prIndexed": True, + "pullRequestId": request.pullRequestId, + "targetBranch": "main", + "sourceRevision": HEAD_REVISION, + "baseRevision": BASE_REVISION, + "baseGenerationManifestSha256": "3" * 64, + "prGenerationFingerprint": "sha256:" + "4" * 64, + "prOverlayGenerationManifestSha256": "5" * 64, + "basePluginFingerprint": "sha256:" + "6" * 64, + "basePluginDescriptorFingerprint": "sha256:" + "7" * 64, + "basePluginImplementationFingerprint": "sha256:" + "8" * 64, + "baseIndexRepresentationFingerprint": "sha256:" + "9" * 64, + } assert report["reviewIdentity"]["targetBranch"] == "main" assert report["reviewIdentity"]["sourceBranch"] == "feature/dry-run" assert report["reviewIdentity"]["headRevision"] == HEAD_REVISION @@ -806,6 +837,17 @@ async def test_full_pipeline_capture_persists_real_context_artifact( assert rag.index_requests assert rag.index_requests[0]["source_revision"] == HEAD_REVISION assert rag.index_requests[0]["base_revision"] == BASE_REVISION + assert rag.requests + assert all( + request["source_revision"] == HEAD_REVISION + and request["base_revision"] == BASE_REVISION + and request["base_generation_manifest_sha256"] == "3" * 64 + and request["pr_generation_fingerprint"] + == "sha256:" + "4" * 64 + and request["pr_overlay_generation_manifest_sha256"] + == "5" * 64 + for request in rag.requests + ) assert { file_info["content_state"] for file_info in rag.index_requests[0]["files"] @@ -1013,6 +1055,13 @@ async def index_with_groups(**kwargs): return { "status": "indexed", "chunks_indexed": 4, + "base_generation_manifest_sha256": "3" * 64, + "generation_fingerprint": "sha256:" + "4" * 64, + "overlay_generation_manifest_sha256": "5" * 64, + "plugin_fingerprint": "sha256:" + "6" * 64, + "plugin_descriptor_fingerprint": "sha256:" + "7" * 64, + "plugin_implementation_fingerprint": "sha256:" + "8" * 64, + "index_representation_fingerprint": "sha256:" + "9" * 64, "review_groups": [ ["src/file_0.py", "src/file_1.py"], ["src/file_1.py", "src/file_2.py"], diff --git a/python-ecosystem/inference-orchestrator/tests/test_quality_capture.py b/python-ecosystem/inference-orchestrator/tests/test_quality_capture.py index 0f3431cca..f3d99f0b7 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_quality_capture.py +++ b/python-ecosystem/inference-orchestrator/tests/test_quality_capture.py @@ -48,6 +48,23 @@ def _request(**updates): return ReviewRequestDto(**values) +def _revision_binding(): + return { + "prIndexed": True, + "pullRequestId": 12, + "targetBranch": "main", + "sourceRevision": "b" * 40, + "baseRevision": "a" * 40, + "baseGenerationManifestSha256": "c" * 64, + "prGenerationFingerprint": "sha256:" + "d" * 64, + "prOverlayGenerationManifestSha256": "e" * 64, + "basePluginFingerprint": "sha256:" + "1" * 64, + "basePluginDescriptorFingerprint": "sha256:" + "2" * 64, + "basePluginImplementationFingerprint": "sha256:" + "3" * 64, + "baseIndexRepresentationFingerprint": "sha256:" + "4" * 64, + } + + class _FakeDelegate: def __init__(self): self.calls = [] @@ -82,6 +99,9 @@ async def ainvoke(self, input_data, **kwargs): "output_tokens": 2, }, }]], + "llm_output": { + "model_name": "provider-resolved-review-model", + }, }) return { "answer": "captured", @@ -114,6 +134,26 @@ def test_non_allowlisted_project_is_not_captured(capture_environment): assert list(capture_environment.iterdir()) == [] +def test_provider_model_identity_uses_metadata_not_tool_arguments(): + response = { + "llm_output": {"model_name": "provider/resolved"}, + "generations": [[{ + "message": { + "tool_calls": [{ + "args": {"model": "domain-object-name"}, + }], + "response_metadata": { + "model": "provider/resolved", + }, + }, + }]], + } + + assert quality_capture._provider_reported_models(response) == [ + "provider/resolved" + ] + + @pytest.mark.asyncio async def test_capture_records_exact_model_boundary_and_redacts_credentials( capture_environment, @@ -125,7 +165,12 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( event_callback = session.wrap_event_callback(forwarded_events.append) capped = llm.model_copy(update={"max_tokens": 321}) - structured = capped.with_structured_output(dict, include_raw=True) + structured = capped.with_structured_output( + dict, + include_raw=True, + method="json_schema", + strict=True, + ) result = await structured.ainvoke("Review proprietary_source safely") event_callback({ "type": "status", @@ -161,6 +206,7 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( "semanticDisabled": False, "exactEvidenceIds": 2, }, + "revisionBinding": _revision_binding(), }) response = {"result": {"issues": [{"file": "private.py", "title": "Example"}]}} await session.complete(response) @@ -179,6 +225,9 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( "registered": 1, "completed": 1, } + assert artifact["pipelineEvidence"]["revisionBinding"] == ( + _revision_binding() + ) assert artifact["pipelineEvidenceDigest"] assert forwarded_events[0]["state"] == "review_evidence_completed" assert artifact["calls"][0]["status"] == "completed" @@ -186,9 +235,19 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( "Review proprietary_source safely" ) assert artifact["calls"][0]["modelBindings"]["max_tokens"] == 321 + assert artifact["calls"][0]["modelBindings"]["structured_output"] == { + "include_raw": True, + "options": { + "method": "json_schema", + "strict": True, + }, + } assert artifact["calls"][0]["response"]["answer"] == "captured" assert artifact["calls"][0]["providerCallCountSource"] == "callback" assert artifact["calls"][0]["providerEvents"][0]["response"]["generations"] + assert artifact["calls"][0]["providerEvents"][0][ + "providerReportedModels" + ] == ["provider-resolved-review-model"] assert artifact["request"]["rawDiff"].endswith("proprietary_source = True") assert artifact["request"]["aiCustomParameters"]["temperature"] == 0.2 assert artifact["request"]["aiCustomParameters"]["default_headers"]["X-Trace"] == ( @@ -209,6 +268,14 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( assert len(artifact["modeIdentity"]) == 64 assert artifact["captureDigest"] assert artifact["resultDigest"] + receipt = session.receipt() + assert receipt["requestedModel"] == "review-model" + assert receipt["providerReportedModels"] == [ + "provider-resolved-review-model" + ] + assert receipt["providerModelEvidenceComplete"] is True + assert receipt["calls"][0]["stage"] + assert receipt["receiptDigest"] assert "provider-secret" not in serialized assert "header-secret" not in serialized assert "oauth-secret" not in serialized @@ -216,6 +283,35 @@ async def test_capture_records_exact_model_boundary_and_redacts_credentials( assert (os.stat(session.path).st_mode & 0o777) == 0o600 +@pytest.mark.asyncio +async def test_capture_records_tool_binding_options(capture_environment): + session = create_quality_capture_session(_request()) + delegate = _FakeDelegate() + llm = ReviewQualityCaptureLLM(delegate, session) + + bound = llm.bind_tools( + [{"name": "lookup", "description": "Read source evidence"}], + tool_choice="required", + strict=True, + ) + await bound.ainvoke("Use the declared evidence tool") + await session.complete({"result": {"issues": []}}) + + artifact = json.loads(session.path.read_text(encoding="utf-8")) + assert artifact["calls"][0]["modelBindings"]["tool_binding"] == { + "options": { + "strict": True, + "tool_choice": "required", + }, + } + assert artifact["calls"][0]["tools"] == [ + { + "description": "Read source evidence", + "name": "lookup", + }, + ] + + @pytest.mark.asyncio async def test_capture_marks_missing_or_invalid_terminal_pipeline_evidence( capture_environment, @@ -324,6 +420,12 @@ def wrapped(event): async def complete(self, response, error=None, *, failed=False): completed.append((response, error, failed)) + def receipt(self): + return { + "kind": "review-quality-capture-receipt", + "receiptDigest": "a" * 64, + } + capture = CaptureSpy() async def fake_review( @@ -352,8 +454,21 @@ async def fake_review( ) assert response == {"result": {"issues": []}} - assert observed == [{"state": "review_evidence_completed"}] - assert forwarded == observed + assert observed[0] == {"state": "review_evidence_completed"} + assert observed[1]["state"] == "review_quality_capture_completed" + assert observed[1]["qualityCapture"]["receiptDigest"] == "a" * 64 + assert forwarded == [ + {"state": "review_evidence_completed"}, + { + "type": "status", + "state": "review_quality_capture_completed", + "message": "Review quality capture completed", + "qualityCapture": { + "kind": "review-quality-capture-receipt", + "receiptDigest": "a" * 64, + }, + }, + ] assert completed == [(response, None, False)] diff --git a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py index cbf2d7aea..510cac40a 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_rag_client.py +++ b/python-ecosystem/inference-orchestrator/tests/test_rag_client.py @@ -169,6 +169,22 @@ async def test_delete_pr_files_ok(self): assert await c.delete_pr_files("ws", "proj", 1) is True await c.close() + @pytest.mark.asyncio(loop_scope="function") + @respx.mock + async def test_delete_pr_files_uses_exact_generation_target(self): + route = respx.delete("http://rag:8001/index/pr-files/ws/proj/1").mock( + return_value=httpx.Response(200, json={"status": "deleted"}) + ) + c = RagClient(base_url="http://rag:8001", enabled=True) + + assert await c.delete_pr_files( + "ws", "proj", 1, collection_target="cc_w1_p2_branch_generation" + ) is True + assert route.calls.last.request.url.params[ + "collection_target" + ] == "cc_w1_p2_branch_generation" + await c.close() + # ── Error handling ─────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_rag_client_duplication_unit.py b/python-ecosystem/inference-orchestrator/tests/test_rag_client_duplication_unit.py index 573161f05..997d07574 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_rag_client_duplication_unit.py +++ b/python-ecosystem/inference-orchestrator/tests/test_rag_client_duplication_unit.py @@ -3,7 +3,7 @@ import pytest -from service.rag.rag_client import RagClient +from service.rag.rag_client import RagClient, RagRetrievalError class _FakeResponse: @@ -59,6 +59,27 @@ async def get_client(): assert time.perf_counter() - started < 0.5 +@pytest.mark.asyncio(loop_scope="function") +async def test_revision_bound_duplication_timeout_is_propagated(monkeypatch): + monkeypatch.setenv("REVIEW_DUPLICATION_RAG_QUERY_TIMEOUT_SECONDS", "0.1") + client = RagClient(base_url="http://rag", enabled=True) + + async def get_client(): + return _SlowSearchClient() + + client._get_client = get_client + + with pytest.raises(RagRetrievalError, match="timed out"): + await client.search_for_duplicates( + workspace="ws", + project="proj", + branch="main", + queries=["find duplicate implementation"], + repository_revision="a" * 40, + repository_generation_manifest_sha256="b" * 64, + ) + + @pytest.mark.asyncio(loop_scope="function") async def test_duplication_search_runs_queries_concurrently(monkeypatch): monkeypatch.setenv("REVIEW_DUPLICATION_RAG_QUERY_TIMEOUT_SECONDS", "1") diff --git a/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py index 7d7558c79..405c33c6e 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_review_service_helpers.py @@ -7,7 +7,10 @@ import pytest from unittest.mock import MagicMock, patch -from service.review.review_service import ReviewService +from service.review.review_service import ( + ReviewService, + allow_unbound_global_rag_fallback, +) @pytest.fixture @@ -143,6 +146,26 @@ def test_enabled_project_uses_enabled_shared_client(self, service): assert service._rag_client_for_request(request) is service.rag_client + def test_exact_pr_review_disables_unbound_global_fallback(self): + request = MagicMock( + pullRequestId=42, + baseCommitHash="a" * 40, + currentCommitHash="b" * 40, + commitHash=None, + ) + + assert allow_unbound_global_rag_fallback(request) is False + + def test_non_pr_request_can_use_global_fallback(self): + request = MagicMock( + pullRequestId=None, + baseCommitHash=None, + currentCommitHash="b" * 40, + commitHash=None, + ) + + assert allow_unbound_global_rag_fallback(request) is True + # ── _create_llm ────────────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py b/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py index 84f836333..461da71ff 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_1_file_review.py @@ -1146,6 +1146,21 @@ def _request(self): request.enrichmentData = None return request + def _exact_request(self): + request = self._request() + request.currentCommitHash = "a" * 40 + request.commitHash = "a" * 40 + request.baseCommitHash = "b" * 40 + request.ragBaseGenerationManifestSha256 = "c" * 64 + request.ragPrGenerationFingerprint = "d" * 64 + request.ragPrOverlayGenerationManifestSha256 = "e" * 64 + request.rawDiff = "" + request.deltaDiff = None + request.taskContext = None + request.projectRules = [] + request.previousCodeAnalysisIssues = [] + return request + @pytest.mark.asyncio(loop_scope="function") async def test_missing_target_branch_does_not_query_an_invented_branch(self): request = self._request() @@ -1399,6 +1414,185 @@ async def search_for_duplicates(self, **kwargs): assert state.deterministic_retrieval_states == ["failed"] assert state.semantic_disabled is False + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_deterministic_failure_prevents_review_model_call(self): + class Rag: + async def get_deterministic_context(self, **kwargs): + return { + "status": "error", + "status_code": 500, + "error": "exact retrieval failed", + } + + async def get_pr_context(self, **kwargs): + raise AssertionError("semantic retrieval must not run") + + async def search_for_duplicates(self, **kwargs): + return [] + + batch = [{ + "file": ReviewFile( + path="src/a.py", + focus_areas=["general"], + risk_level="MEDIUM", + ), + "priority": "MEDIUM", + }] + + with patch( + "service.review.orchestrator.stage_1_file_review." + "_invoke_stage_1_batch_llm", + new_callable=AsyncMock, + ) as invoke: + with pytest.raises( + RuntimeError, + match="revision-bound deterministic RAG retrieval failed", + ): + await review_file_batch( + MagicMock(), + self._exact_request(), + batch, + rag_client=Rag(), + prepared_context=Stage1PreparedContext(), + pr_indexed=True, + rag_state=Stage1RagState(), + ) + + invoke.assert_not_awaited() + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_missing_rag_client_prevents_review_model_call(self): + batch = [{ + "file": ReviewFile( + path="src/a.py", + focus_areas=["general"], + risk_level="MEDIUM", + ), + "priority": "MEDIUM", + }] + + with patch( + "service.review.orchestrator.stage_1_file_review." + "_invoke_stage_1_batch_llm", + new_callable=AsyncMock, + ) as invoke: + with pytest.raises( + RuntimeError, + match="requires a RAG client", + ): + await review_file_batch( + MagicMock(), + self._exact_request(), + batch, + rag_client=None, + prepared_context=Stage1PreparedContext(), + pr_indexed=True, + rag_state=Stage1RagState(), + ) + + invoke.assert_not_awaited() + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_partial_deterministic_state_fails_closed(self): + class Rag: + async def get_deterministic_context(self, **kwargs): + return { + "context": { + "chunks": [{"text": "partial context"}], + "_metadata": {"retrieval_state": "partial"}, + } + } + + async def search_for_duplicates(self, **kwargs): + return [] + + with pytest.raises( + RuntimeError, + match="deterministic RAG retrieval is not complete: partial", + ): + await fetch_batch_rag_context( + Rag(), + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=Stage1RagState(), + ) + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_semantic_transport_failure_fails_closed(self): + class Rag: + async def get_deterministic_context(self, **kwargs): + return { + "context": { + "chunks": [], + "_metadata": {"retrieval_state": "complete"}, + } + } + + async def get_pr_context(self, **kwargs): + return { + "status": "error", + "status_code": 503, + "error": "semantic backend unavailable", + } + + async def search_for_duplicates(self, **kwargs): + return [] + + with pytest.raises(RuntimeError, match="semantic backend unavailable"): + await fetch_batch_rag_context( + Rag(), + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=Stage1RagState(), + ) + + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_bound_success_allows_intentional_semantic_disable( + self, + monkeypatch, + ): + import service.review.orchestrator.stage_1_file_review as stage1 + + monkeypatch.setattr(stage1, "SEMANTIC_RAG_FILLER_ENABLED", False) + + class Rag: + async def get_deterministic_context(self, **kwargs): + return { + "context": { + "chunks": [{ + "text": "exact context", + "metadata": {"path": "src/dependency.py"}, + }], + "_metadata": {"retrieval_state": "complete"}, + } + } + + async def get_pr_context(self, **kwargs): + raise AssertionError("semantic retrieval is intentionally disabled") + + async def search_for_duplicates(self, **kwargs): + return [] + + state = Stage1RagState() + result = await fetch_batch_rag_context( + Rag(), + self._exact_request(), + ["src/a.py"], + ["changed line"], + pr_indexed=True, + rag_state=state, + ) + + assert [chunk["text"] for chunk in result["relevant_code"]] == [ + "exact context" + ] + assert state.deterministic_retrieval_states == ["complete"] + assert state.semantic_disabled is False + # ── _deduplicate_pr_stale_chunks ───────────────────────────────── @@ -1860,6 +2054,36 @@ async def test_missing_target_branch_uses_local_grouping_without_rag(self, mock_ assert mock_smart.call_args.kwargs["branches"] == [] assert mock_smart.call_args.kwargs["rag_client"] is None + @patch("service.review.orchestrator.stage_1_file_review.create_smart_batches_async") + @pytest.mark.asyncio(loop_scope="function") + async def test_exact_receipts_disable_unbound_batching_rag(self, mock_smart): + groups = self._make_plan(["a.py"]) + mock_smart.return_value = [[{ + "file": groups[0].files[0], + "priority": "MEDIUM", + }]] + request = MagicMock( + enrichmentData=None, + maxAllowedTokens=200000, + projectWorkspace="ws", + projectNamespace="proj", + ragBaseGenerationManifestSha256="a" * 64, + ragPrGenerationFingerprint="sha256:" + "b" * 64, + ragPrOverlayGenerationManifestSha256="c" * 64, + ) + request.get_rag_branch.return_value = "main" + request.get_rag_base_branch.return_value = "main" + + result = await create_smart_batches_wrapper( + file_groups=groups, + processed_diff=MagicMock(), + request=request, + rag_client=MagicMock(), + ) + + assert result == mock_smart.return_value + assert mock_smart.call_args.kwargs["rag_client"] is None + class TestStage1Scheduling: @pytest.mark.asyncio(loop_scope="function") diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_2_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_stage_2_helpers.py index c5ce2621f..a330e0f4a 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_2_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_2_helpers.py @@ -171,6 +171,40 @@ async def test_cross_module_context_does_not_guess_main_without_target_branch(): rag.search_for_duplicates.assert_not_awaited() +@pytest.mark.asyncio(loop_scope="function") +async def test_revision_bound_cross_module_transport_failure_fails_open(): + request = SimpleNamespace( + baseCommitHash="a" * 40, + ragBaseGenerationManifestSha256="b" * 64, + changedFiles=["src/service.py"], + prTitle="Change service", + projectWorkspace="ws", + projectNamespace="project", + get_rag_branch=lambda: "main", + get_rag_base_branch=lambda: "main", + ) + rag = SimpleNamespace( + search_for_duplicates=AsyncMock( + side_effect=ConnectionError("transport unavailable") + ), + ) + + assert await _fetch_cross_module_context(rag, request) == "" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_cross_module_context_skips_partial_generation_lease(): + request = SimpleNamespace( + baseCommitHash="a" * 40, + ragBaseGenerationManifestSha256=None, + ) + rag = SimpleNamespace(search_for_duplicates=AsyncMock()) + + assert await _fetch_cross_module_context(rag, request) == "" + + rag.search_for_duplicates.assert_not_awaited() + + # ── _slim_issues_for_stage_2 ──────────────────────────────── class TestSlimIssues: diff --git a/python-ecosystem/rag-pipeline/requirements.local.txt b/python-ecosystem/rag-pipeline/requirements.local.txt index 7e48bdc21..0ea464a50 100644 --- a/python-ecosystem/rag-pipeline/requirements.local.txt +++ b/python-ecosystem/rag-pipeline/requirements.local.txt @@ -10,7 +10,9 @@ llama-index-core==0.13.0 # LlamaIndex extensions llama-index-embeddings-openai>=0.3.0 -llama-index-vector-stores-qdrant>=0.5.0 +# Keep the adapter and client as one tested compatibility pair. qdrant-client +# 1.19 removed IDF_EMBEDDING_MODELS while this adapter still imports it. +llama-index-vector-stores-qdrant==0.10.2 llama-index-llms-openai>=0.3.0 # LangChain text splitters (language-aware code splitting) @@ -36,7 +38,7 @@ tree-sitter-ruby==0.23.1 tree-sitter-php==0.24.1 # Qdrant for vector storage -qdrant-client>=1.7.0,<2.0.0 +qdrant-client==1.16.2 # Redis for async queue redis>=5.0.0,<6.0.0 diff --git a/python-ecosystem/rag-pipeline/requirements.txt b/python-ecosystem/rag-pipeline/requirements.txt index 0ad2b000a..b11a15ea5 100644 --- a/python-ecosystem/rag-pipeline/requirements.txt +++ b/python-ecosystem/rag-pipeline/requirements.txt @@ -10,7 +10,9 @@ llama-index-core==0.13.0 # LlamaIndex extensions llama-index-embeddings-openai>=0.3.0 -llama-index-vector-stores-qdrant>=0.5.0 +# Keep the adapter and client as one tested compatibility pair. qdrant-client +# 1.19 removed IDF_EMBEDDING_MODELS while this adapter still imports it. +llama-index-vector-stores-qdrant==0.10.2 llama-index-llms-openai>=0.3.0 # LangChain text splitters (language-aware code splitting) @@ -36,7 +38,7 @@ tree-sitter-ruby==0.23.1 tree-sitter-php==0.24.1 # Qdrant for vector storage -qdrant-client>=1.7.0,<2.0.0 +qdrant-client==1.16.2 # Redis for async queue redis>=5.0.0,<6.0.0 diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py index 896ef595a..21922f543 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/models.py @@ -39,6 +39,13 @@ class IndexRequest(BaseModel): project: str branch: str commit: str + source_tree_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) + publish_branch_alias: bool = False + publish_legacy_project_alias: bool = False preserve_other_branches: bool = False cleanup_repo_path: bool = False include_patterns: Optional[List[str]] = None @@ -102,6 +109,43 @@ def validate_file_paths(cls, v: List[str]) -> List[str]: return _validate_file_paths(v) +class AdvanceGenerationRequest(BaseModel): + updated_file_paths: List[str] = Field(default_factory=list) + deleted_file_paths: List[str] = Field(default_factory=list) + repo_base: Optional[str] = None + workspace: str + project: str + branch: str + source_commit: str + commit: str + source_tree_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + source_collection_target: str = Field(min_length=1) + collection_target: str = Field(min_length=1) + publish_branch_alias: bool = False + publish_legacy_project_alias: bool = False + + @field_validator("repo_base") + @classmethod + def validate_repo_base(cls, v: Optional[str]) -> Optional[str]: + return _validate_repo_path(v) if v is not None else None + + @field_validator("updated_file_paths", "deleted_file_paths") + @classmethod + def validate_file_paths(cls, v: List[str]) -> List[str]: + return _validate_file_paths(v) + + +class GenerationAliasPublicationRequest(BaseModel): + """Repair the readable aliases of one already sealed generation.""" + workspace: str + project: str + branch: str + commit: str + collection_target: str = Field(min_length=1) + publish_branch_alias: bool = True + publish_legacy_project_alias: bool = False + + class DeleteBranchRequest(BaseModel): workspace: str project: str @@ -126,6 +170,29 @@ def validate_branch_names(cls, value: Optional[List[str]]) -> Optional[List[str] return value +class RevisionPreflightResponse(BaseModel): + workspace: str + project: str + branch: str + commit: str + point_count: int = Field(gt=0) + repository_revision: str + repository_facts_sha256: str + plugin_ids: List[str] + plugin_fingerprint: str + plugin_descriptor_fingerprint: str + plugin_implementation_fingerprint: str + index_representation_fingerprint: str + generation_schema: str + generation_member_count: int = Field(gt=0) + generation_members_sha256: str + generation_manifest_sha256: str + source_tree_sha256: str + index_include_patterns: List[str] + index_exclude_patterns: List[str] + index_selection_policy_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + class EstimateRequest(BaseModel): repo_path: str include_patterns: Optional[List[str]] = None @@ -155,6 +222,16 @@ class QueryRequest(BaseModel): branch: str top_k: Optional[int] = 10 filter_language: Optional[str] = None + repository_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + repository_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) class PRContextRequest(BaseModel): @@ -172,6 +249,29 @@ class PRContextRequest(BaseModel): deleted_files: Optional[List[str]] = Field(default_factory=list) pr_number: Optional[int] = None all_pr_changed_files: Optional[List[str]] = Field(default_factory=list) + source_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + base_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + base_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + pr_generation_fingerprint: Optional[str] = Field( + default=None, + pattern=r"^sha256:[0-9a-f]{64}$", + ) + pr_overlay_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) @field_validator('changed_files') @classmethod @@ -205,6 +305,29 @@ class DeterministicContextRequest(BaseModel): description="Extra type/function names to look up (from AST enrichment: extends, implements, calls). " "Injected directly into Step 2 definition lookup alongside Qdrant-extracted identifiers." ) + source_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + base_revision: Optional[str] = Field( + default=None, + min_length=1, + max_length=200, + ) + base_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + pr_generation_fingerprint: Optional[str] = Field( + default=None, + pattern=r"^sha256:[0-9a-f]{64}$", + ) + pr_overlay_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) # ── Parse models ── @@ -278,6 +401,11 @@ class PRIndexRequest(BaseModel): plugin_fingerprint: str = "sha256:" + "0" * 64 plugin_descriptor_fingerprint: str = "sha256:" + "0" * 64 files: List[PRFileInfo] + base_generation_manifest_sha256: Optional[str] = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) + collection_target: Optional[str] = Field(default=None, min_length=1) # ── Vector storage inspection models ── diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py index 57a3f9e4f..918a612cc 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py @@ -1,11 +1,17 @@ """Index and branch management endpoints.""" +import json import logging +from queue import Queue +from threading import Thread from typing import List -from fastapi import APIRouter, HTTPException, BackgroundTasks +from fastapi import APIRouter, HTTPException, BackgroundTasks, Query +from fastapi.responses import StreamingResponse from ...models.config import IndexStats from ..models import ( IndexRequest, UpdateFilesRequest, DeleteFilesRequest, ApplyChangesRequest, + AdvanceGenerationRequest, + GenerationAliasPublicationRequest, DeleteBranchRequest, CleanupStaleBranchesRequest, EstimateRequest, EstimateResponse, ) @@ -46,7 +52,7 @@ def estimate_repository(request: EstimateRequest): file_count, estimated_chunks = index_manager.estimate_repository_size( repo_path=request.repo_path, include_patterns=request.include_patterns, - exclude_patterns=request.exclude_patterns + exclude_patterns=request.exclude_patterns, ) within_limits = True @@ -87,6 +93,17 @@ def index_repository(request: IndexRequest, background_tasks: BackgroundTasks): """Index entire repository.""" _, index_manager = _get_singletons() try: + optional_generation_args = {} + source_tree_sha256 = getattr(request, "source_tree_sha256", None) + collection_target = getattr(request, "collection_target", None) + if isinstance(source_tree_sha256, str) and source_tree_sha256: + optional_generation_args["source_tree_sha256"] = source_tree_sha256 + if isinstance(collection_target, str) and collection_target: + optional_generation_args["collection_target"] = collection_target + if getattr(request, "publish_branch_alias", False) is True: + optional_generation_args["publish_branch_alias"] = True + if getattr(request, "publish_legacy_project_alias", False) is True: + optional_generation_args["publish_legacy_project_alias"] = True stats = index_manager.index_repository( repo_path=request.repo_path, workspace=request.workspace, @@ -95,7 +112,8 @@ def index_repository(request: IndexRequest, background_tasks: BackgroundTasks): commit=request.commit, preserve_other_branches=request.preserve_other_branches, include_patterns=request.include_patterns, - exclude_patterns=request.exclude_patterns + exclude_patterns=request.exclude_patterns, + **optional_generation_args, ) return stats except ValueError as e: @@ -110,6 +128,71 @@ def index_repository(request: IndexRequest, background_tasks: BackgroundTasks): raise HTTPException(status_code=500, detail=str(e)) +@router.post("/index/repository/stream") +def index_repository_stream(request: IndexRequest): + """Index one repository and stream observable batch progress as SSE. + + The ordinary endpoint remains the stable JSON contract. This endpoint is + intentionally only an observability transport: it runs the same index + operation and forwards optional progress events without making progress + delivery a prerequisite for a successful snapshot. + """ + _, index_manager = _get_singletons() + + def event_stream(): + events: Queue[tuple[str, object]] = Queue() + + def progress(event: dict) -> None: + events.put(("progress", event)) + + def run_index() -> None: + try: + optional_generation_args = {} + if request.source_tree_sha256: + optional_generation_args["source_tree_sha256"] = ( + request.source_tree_sha256 + ) + if request.collection_target: + optional_generation_args["collection_target"] = ( + request.collection_target + ) + if getattr(request, "publish_branch_alias", False) is True: + optional_generation_args["publish_branch_alias"] = True + if getattr(request, "publish_legacy_project_alias", False) is True: + optional_generation_args["publish_legacy_project_alias"] = True + stats = index_manager.index_repository( + repo_path=request.repo_path, + workspace=request.workspace, + project=request.project, + branch=request.branch, + commit=request.commit, + preserve_other_branches=request.preserve_other_branches, + include_patterns=request.include_patterns, + exclude_patterns=request.exclude_patterns, + progress_callback=progress, + **optional_generation_args, + ) + events.put(("complete", stats.model_dump(mode="json"))) + except Exception as exception: + logger.error("Error indexing repository with progress: %s", exception) + events.put(("error", {"message": str(exception)})) + + Thread(target=run_index, name="rag-index-progress", daemon=True).start() + while True: + event_type, payload = events.get() + if event_type == "progress": + event = {"type": "progress", **payload} + elif event_type == "complete": + event = {"type": "complete", "result": payload} + else: + event = {"type": "error", **payload} + yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" + if event_type in {"complete", "error"}: + break + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + @router.post("/index/update-files", response_model=IndexStats) def update_files(request: UpdateFilesRequest): """Update specific files in index.""" @@ -195,6 +278,75 @@ def apply_changes(request: ApplyChangesRequest): raise HTTPException(status_code=500, detail=str(e)) +@router.post("/index/advance-generation", response_model=IndexStats) +def advance_generation(request: AdvanceGenerationRequest): + """Build one immutable target generation from an exact sealed source.""" + _, index_manager = _get_singletons() + if request.repo_base is None: + raise HTTPException( + status_code=422, + detail="repo_base is required to attest the target source tree", + ) + try: + return index_manager.advance_generation( + source_collection_target=request.source_collection_target, + target_collection_target=request.collection_target, + source_commit=request.source_commit, + source_tree_sha256=request.source_tree_sha256, + updated_file_paths=request.updated_file_paths, + deleted_file_paths=request.deleted_file_paths, + repo_base=request.repo_base, + workspace=request.workspace, + project=request.project, + branch=request.branch, + commit=request.commit, + publish_branch_alias=request.publish_branch_alias, + publish_legacy_project_alias=request.publish_legacy_project_alias, + ) + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=422, detail=str(e)) + except MutationLeaseUnavailable as e: + raise HTTPException(status_code=409, detail=str(e)) + except MutationCoordinationUnavailable as e: + raise HTTPException(status_code=503, detail=str(e)) + except Exception as e: + logger.error(f"Error advancing repository generation: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/index/generation-aliases") +def publish_generation_aliases(request: GenerationAliasPublicationRequest): + """Repair readable aliases for a completed immutable generation. + + This is deliberately separate from indexing. Indexing publishes all aliases + atomically; this endpoint makes deployments backward-compatible by repairing + active generations that predate readable branch aliases. + """ + _, index_manager = _get_singletons() + try: + aliases = index_manager.publish_generation_aliases( + workspace=request.workspace, + project=request.project, + branch=request.branch, + commit=request.commit, + collection_target=request.collection_target, + publish_branch_alias=request.publish_branch_alias, + publish_legacy_project_alias=request.publish_legacy_project_alias, + ) + return {"status": "published", "aliases": aliases} + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) + except MutationLeaseUnavailable as e: + raise HTTPException(status_code=409, detail=str(e)) + except MutationCoordinationUnavailable as e: + raise HTTPException(status_code=503, detail=str(e)) + except Exception as e: + logger.error("Error publishing readable generation aliases: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + @router.delete("/index/{workspace}/{project}/{branch}") def delete_index(workspace: str, project: str, branch: str): """Delete entire index.""" @@ -214,11 +366,18 @@ def delete_index(workspace: str, project: str, branch: str): # ── Branch management ── @router.delete("/index/{workspace}/{project}/branch/{branch}") -def delete_branch(workspace: str, project: str, branch: str): +def delete_branch( + workspace: str, + project: str, + branch: str, + collection_target: str | None = Query(default=None), +): """Delete all points for a specific branch from the project collection.""" _, index_manager = _get_singletons() try: - success = index_manager.delete_branch(workspace, project, branch) + success = index_manager.delete_branch( + workspace, project, branch, collection_target=collection_target + ) if success: return { "status": "success", @@ -229,6 +388,8 @@ def delete_branch(workspace: str, project: str, branch: str): "status": "not_found", "message": f"Branch '{branch}' not found or collection doesn't exist" } + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) except MutationLeaseUnavailable as e: raise HTTPException(status_code=409, detail=str(e)) except MutationCoordinationUnavailable as e: diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/inspect.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/inspect.py index 1d9a66ab6..df9a0de23 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/inspect.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/inspect.py @@ -34,6 +34,9 @@ "repository_facts_state", "facts_part", "facts_parts", "facts_content_sha256", "plugin_ids", "plugin_fingerprint", "plugin_descriptor_fingerprint", "plugin_implementation_fingerprint", + "repository_generation_manifest", "generation_schema", + "generation_member_count", "generation_members_sha256", + "generation_manifest_sha256", "generation_member_sha256", "indexed_at", "fragment_of", "text", "_node_content", ] GRAPH_TEXT_LIMIT = 280 @@ -191,6 +194,8 @@ def _node_title(payload: Dict[str, Any]) -> str: return f"{plugin}: {kind}" + (f" — {source}" if source else "") if payload.get("repository_facts_state"): return "Repository detection facts" + if payload.get("repository_generation_manifest"): + return "Repository generation manifest" if payload.get("repository_snapshot"): return ( f"{payload.get('snapshot_plugin') or 'plugin'}: " @@ -218,6 +223,8 @@ def _node_kind(payload: Dict[str, Any]) -> str: return "repository_snapshot" if payload.get("repository_facts_state"): return "repository_facts" + if payload.get("repository_generation_manifest"): + return "repository_generation_manifest" if payload.get("node_type"): return str(payload["node_type"]) if payload.get("content_type"): diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py index cba269d31..39bd10905 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py @@ -1,7 +1,7 @@ """PR file indexing endpoints.""" import logging from datetime import datetime, timezone -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Query from llama_index.core import Document as LlamaDocument from qdrant_client.models import Filter, FieldCondition, MatchValue @@ -28,6 +28,10 @@ from ...core.pr_overlay_representation import ( PR_OVERLAY_REPRESENTATION_PAYLOAD_KEY, ) +from ...core.pr_overlay_manifest import ( + read_pr_overlay_generation, +) +from ...core.revision_binding import require_repository_generation logger = logging.getLogger(__name__) router = APIRouter(tags=["pr"]) @@ -114,8 +118,59 @@ def index_pr_files(request: PRIndexRequest): mutation_lease = None try: mutation_lease = mutation_context.__enter__() - collection_name = index_manager._get_project_collection_name( - request.workspace, request.project + base_receipt = None + target_branch = request.base_branch or request.branch + requested_collection_target = getattr( + request, "collection_target", None + ) + requested_base_manifest = getattr( + request, "base_generation_manifest_sha256", None + ) + if not isinstance(requested_collection_target, str): + requested_collection_target = None + if not isinstance(requested_base_manifest, str): + requested_base_manifest = None + exact_binding = bool( + request.base_revision + and (requested_collection_target or requested_base_manifest) + ) + if exact_binding: + base_receipt = require_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=target_branch, + revision=request.base_revision, + generation_manifest_sha256=( + requested_base_manifest + ), + collection_target=requested_collection_target, + ) + collection_name = ( + base_receipt["_collection_target"] + if base_receipt + else requested_collection_target + or index_manager._get_project_collection_name( + request.workspace, request.project + ) + ) + base_generation_receipt = ( + { + "base_generation_manifest_sha256": base_receipt[ + "generation_manifest_sha256" + ], + "plugin_fingerprint": base_receipt["plugin_fingerprint"], + "plugin_descriptor_fingerprint": base_receipt[ + "plugin_descriptor_fingerprint" + ], + "plugin_implementation_fingerprint": base_receipt[ + "plugin_implementation_fingerprint" + ], + "index_representation_fingerprint": base_receipt[ + "index_representation_fingerprint" + ], + } + if base_receipt else {} ) index_manager._ensure_collection_exists(collection_name) @@ -147,7 +202,6 @@ def index_pr_files(request: PRIndexRequest): # Recover the exact plugin-owned repository state from the PR target # branch. The host never interprets the snapshots; it only validates, # overlays changed artifacts, and asks the selected plugins to rebuild. - target_branch = request.base_branch or request.branch representation_fingerprint = ( index_manager.index_representation_fingerprint ) @@ -322,6 +376,10 @@ def index_pr_files(request: PRIndexRequest): base_branch=target_branch, source_revision=request.source_revision, base_revision=request.base_revision, + base_generation_manifest_sha256=( + base_receipt["generation_manifest_sha256"] + if base_receipt else "" + ), files=request.files, requested_plugin_ids=requested_plugin_ids, repository_plugin_ids=repository_plugins, @@ -344,9 +402,32 @@ def index_pr_files(request: PRIndexRequest): ), snapshots=snapshots, ) - if is_complete_reusable_generation( - old_pr_points, - generation_fingerprint, + reusable_receipt = ( + read_pr_overlay_generation( + index_manager.qdrant_client, + collection_name, + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=request.branch, + base_branch=target_branch, + source_revision=request.source_revision, + base_revision=request.base_revision, + base_generation_manifest_sha256=base_receipt[ + "generation_manifest_sha256" + ], + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + ) + if base_receipt else None + ) + if reusable_receipt is not None or ( + base_receipt is None + and is_complete_reusable_generation( + old_pr_points, generation_fingerprint + ) ): architecture_points = sum( 1 @@ -364,12 +445,14 @@ def index_pr_files(request: PRIndexRequest): ) return { "status": "reused", + **base_generation_receipt, "pr_number": request.pr_number, "files_processed": len(request.files), "chunks_indexed": len(old_pr_points), "chunks_failed": 0, "architecture_packets_indexed": architecture_points, "generation_fingerprint": generation_fingerprint, + **(reusable_receipt or {}), "overlay_representation_fingerprint": ( overlay_representation_fingerprint ), @@ -497,6 +580,11 @@ def index_pr_files(request: PRIndexRequest): ) chunk.metadata["pr_source_revision"] = request.source_revision chunk.metadata["pr_base_revision"] = request.base_revision + if base_receipt: + chunk.metadata["pr_base_generation_manifest_sha256"] = ( + base_receipt["generation_manifest_sha256"] + ) + chunk.metadata["pr_overlay_base_branch"] = target_branch chunk.metadata["indexed_at"] = datetime.now(timezone.utc).isoformat() point_id_branch = f"__pr__/{request.pr_number}/{request.branch}" @@ -593,16 +681,71 @@ def index_pr_files(request: PRIndexRequest): ) node.metadata["pr_source_revision"] = request.source_revision node.metadata["pr_base_revision"] = request.base_revision + if base_receipt: + node.metadata["pr_base_generation_manifest_sha256"] = ( + base_receipt["generation_manifest_sha256"] + ) + node.metadata["pr_overlay_base_branch"] = target_branch node.metadata["indexed_at"] = datetime.now(timezone.utc).isoformat() - successful = index_manager._file_ops._replace_points( - [*chunks, *architecture_nodes], - old_pr_points, - collection_name, - request.workspace, - request.project, - point_id_branch, - mutation_lease.assert_owned, - ) + overlay_receipt = {} + if generation_fingerprint and base_receipt: + identity_metadata = { + "plugin_ids": list( + capabilities.repository_plugins + if capabilities is not None else stored_plugin_ids + ), + "plugin_fingerprint": ( + capabilities.fingerprint + if capabilities is not None else stored_fingerprint + ) or ZERO_FINGERPRINT, + "plugin_descriptor_fingerprint": ( + capabilities.descriptor_fingerprint + if capabilities is not None + else _stored_descriptor_fingerprint + ) or ZERO_FINGERPRINT, + "plugin_implementation_fingerprint": ( + implementation_fingerprint or ZERO_FINGERPRINT + ), + INDEX_REPRESENTATION_PAYLOAD_KEY: representation_fingerprint, + PR_OVERLAY_REPRESENTATION_PAYLOAD_KEY: ( + overlay_representation_fingerprint + ), + } + + successful, overlay_receipt = ( + index_manager._file_ops.replace_pr_overlay_generation( + [*chunks, *architecture_nodes], + old_pr_points, + collection_name, + request.workspace, + request.project, + point_id_branch, + mutation_lease.assert_owned, + pr_number=request.pr_number, + branch=request.branch, + base_branch=target_branch, + source_revision=request.source_revision, + base_revision=request.base_revision, + base_generation_manifest_sha256=base_receipt[ + "generation_manifest_sha256" + ], + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + identity_metadata=identity_metadata, + ) + ) + else: + successful = index_manager._file_ops._replace_points( + [*chunks, *architecture_nodes], + old_pr_points, + collection_name, + request.workspace, + request.project, + point_id_branch, + mutation_lease.assert_owned, + ) skipped_points = ( len(chunks) + len(architecture_nodes) - successful ) @@ -617,6 +760,7 @@ def index_pr_files(request: PRIndexRequest): return { "status": "indexed", + **base_generation_receipt, "pr_number": request.pr_number, "files_processed": len(request.files), "chunks_indexed": successful, @@ -625,6 +769,7 @@ def index_pr_files(request: PRIndexRequest): "skipped_files": list(split_skipped_paths), "architecture_packets_indexed": len(architecture_nodes), "generation_fingerprint": generation_fingerprint, + **overlay_receipt, "overlay_representation_fingerprint": ( overlay_representation_fingerprint ), @@ -664,7 +809,12 @@ def index_pr_files(request: PRIndexRequest): @router.delete("/index/pr-files/{workspace}/{project}/{pr_number}") -def delete_pr_files(workspace: str, project: str, pr_number: int): +def delete_pr_files( + workspace: str, + project: str, + pr_number: int, + collection_target: str | None = Query(default=None), +): """Delete all indexed points for a specific PR.""" index_manager = _get_index_manager() try: @@ -673,7 +823,10 @@ def delete_pr_files(workspace: str, project: str, pr_number: int): project, "delete-pr-overlay", ) as lease: - collection_name = index_manager._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or index_manager._get_project_collection_name(workspace, project) + ) if not index_manager._collection_manager.collection_exists(collection_name): return {"status": "skipped", "message": "Collection does not exist"} diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py index 9e9c08948..826e38cad 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py @@ -5,6 +5,15 @@ from qdrant_client.models import Filter, FieldCondition, MatchAny, MatchValue from ..models import QueryRequest, PRContextRequest, DeterministicContextRequest +from ...core.revision_binding import ( + require_repository_generation, + require_same_repository_generation, +) +from ...core.repository_overlay import IncrementalIndexPreconditionError +from ...core.pr_overlay_manifest import ( + PR_OVERLAY_MANIFEST_PAYLOAD_KEY, + read_pr_overlay_generation, +) logger = logging.getLogger(__name__) router = APIRouter(tags=["query"]) @@ -15,6 +24,11 @@ def _get_singletons(): return index_manager, query_service +def _optional_string(value) -> Optional[str]: + """Ignore absent/loose legacy DTO attributes instead of binding them.""" + return value if isinstance(value, str) and value else None + + def _authoritative_pr_branch(request: PRContextRequest) -> Optional[str]: """Return target-branch truth for hybrid PR retrieval. @@ -31,17 +45,50 @@ def _authoritative_pr_branch(request: PRContextRequest) -> Optional[str]: @router.post("/query/search") def semantic_search(request: QueryRequest): """Perform semantic search.""" - _, query_service = _get_singletons() + index_manager, query_service = _get_singletons() try: + repository_revision = _optional_string( + request.repository_revision + ) + generation_manifest = _optional_string( + request.repository_generation_manifest_sha256 + ) + collection_target = _optional_string(request.collection_target) + receipt = None + if repository_revision: + receipt = require_repository_generation( + index_manager=index_manager, + workspace=request.workspace, + project=request.project, + branch=request.branch, + revision=repository_revision, + generation_manifest_sha256=generation_manifest, + collection_target=collection_target, + ) results = query_service.semantic_search( query=request.query, workspace=request.workspace, project=request.project, branch=request.branch, top_k=request.top_k, - filter_language=request.filter_language + filter_language=request.filter_language, + expected_revision=repository_revision, + collection_target=( + receipt["_collection_target"] if receipt else collection_target + ), ) + if receipt: + require_same_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=request.branch, + revision=repository_revision, + receipt=receipt, + ) return {"results": results} + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) except Exception as e: logger.error(f"Error performing search: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -75,10 +122,39 @@ def get_pr_context(request: PRContextRequest): } } + source_revision = _optional_string(request.source_revision) + base_revision = _optional_string(request.base_revision) + base_generation_manifest = _optional_string( + request.base_generation_manifest_sha256 + ) + pr_generation_fingerprint = _optional_string( + request.pr_generation_fingerprint + ) + pr_overlay_manifest = _optional_string( + request.pr_overlay_generation_manifest_sha256 + ) + collection_target = _optional_string(request.collection_target) + receipt = None + overlay_receipt = None + if base_revision: + receipt = require_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=authoritative_branch, + revision=base_revision, + generation_manifest_sha256=base_generation_manifest, + collection_target=collection_target, + ) pr_results = [] - collection_name = index_manager._get_project_collection_name( - request.workspace, - request.project, + collection_name = ( + receipt["_collection_target"] + if receipt + else collection_target + or index_manager._get_project_collection_name( + request.workspace, + request.project, + ) ) preflight_branches = [authoritative_branch] if query_service._collection_or_alias_exists(collection_name): @@ -89,6 +165,36 @@ def get_pr_context(request: PRContextRequest): # HYBRID MODE: Query PR-indexed data first if pr_number is provided if request.pr_number: + if all(( + source_revision, + base_revision, + base_generation_manifest, + pr_generation_fingerprint, + pr_overlay_manifest, + )): + overlay_receipt = read_pr_overlay_generation( + index_manager.qdrant_client, + collection_name, + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=request.branch or authoritative_branch, + base_branch=authoritative_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + generation_fingerprint=pr_generation_fingerprint, + overlay_representation_fingerprint=( + index_manager.pr_overlay_representation_fingerprint + ), + expected_manifest_sha256=( + pr_overlay_manifest + ), + ) + if overlay_receipt is None: + raise IncrementalIndexPreconditionError( + "requested PR overlay generation is unavailable" + ) pr_results = _query_pr_indexed_data( index_manager=index_manager, query_service=query_service, @@ -98,7 +204,12 @@ def get_pr_context(request: PRContextRequest): changed_files=request.changed_files, query_texts=request.diff_snippets or [], pr_title=request.pr_title, - top_k=request.top_k or 15 + top_k=request.top_k or 15, + collection_target=collection_name, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + pr_generation_fingerprint=pr_generation_fingerprint, ) logger.info(f"Hybrid mode: Found {len(pr_results)} PR-specific chunks for PR #{request.pr_number}") @@ -118,7 +229,12 @@ def get_pr_context(request: PRContextRequest): # branch must not enter the repository branch query a second time. base_branch=None if request.pr_number else request.base_branch, deleted_files=request.deleted_files or [], - exclude_pr_files=(request.all_pr_changed_files or []) if request.pr_number else [] + exclude_pr_files=(request.all_pr_changed_files or []) if request.pr_number else [], + expected_revisions=( + {authoritative_branch: base_revision} + if base_revision else None + ), + collection_target=collection_name, ) # Merge PR results with branch results (PR first, then branch) @@ -155,7 +271,41 @@ def get_pr_context(request: PRContextRequest): "pr_number": request.pr_number } + if receipt: + require_same_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=authoritative_branch, + revision=base_revision, + receipt=receipt, + ) + if request.pr_number and overlay_receipt: + current_overlay = read_pr_overlay_generation( + index_manager.qdrant_client, + collection_name, + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=request.branch or authoritative_branch, + base_branch=authoritative_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + generation_fingerprint=pr_generation_fingerprint, + overlay_representation_fingerprint=( + index_manager.pr_overlay_representation_fingerprint + ), + expected_manifest_sha256=None, + ) + if current_overlay != overlay_receipt: + raise IncrementalIndexPreconditionError( + "PR overlay generation changed while context was retrieved" + ) + return {"context": context} + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) except Exception as e: logger.error(f"Error getting PR context: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -170,7 +320,12 @@ def _query_pr_indexed_data( changed_files: List[str], query_texts: List[str], pr_title: Optional[str], - top_k: int = 15 + top_k: int = 15, + collection_target: Optional[str] = None, + source_revision: Optional[str] = None, + base_revision: Optional[str] = None, + base_generation_manifest_sha256: Optional[str] = None, + pr_generation_fingerprint: Optional[str] = None, ) -> List[Dict]: """ Query PR-indexed chunks from the main collection. @@ -180,9 +335,22 @@ def _query_pr_indexed_data( wasting an embedding call on a fabricated query. """ try: - collection_name = index_manager._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or index_manager._get_project_collection_name(workspace, project) + ) + exact_binding = all(( + source_revision, + base_revision, + base_generation_manifest_sha256, + pr_generation_fingerprint, + )) if not index_manager._collection_manager.collection_exists(collection_name): + if exact_binding: + raise IncrementalIndexPreconditionError( + "revision-bound PR overlay collection is unavailable" + ) return [] query_parts = [] @@ -194,8 +362,21 @@ def _query_pr_indexed_data( pr_filter = Filter( must=[ FieldCondition(key="pr", match=MatchValue(value=True)), - FieldCondition(key="pr_number", match=MatchValue(value=pr_number)) - ] + FieldCondition(key="pr_number", match=MatchValue(value=pr_number)), + *( + [ + FieldCondition(key="pr_source_revision", match=MatchValue(value=source_revision)), + FieldCondition(key="pr_base_revision", match=MatchValue(value=base_revision)), + FieldCondition(key="pr_base_generation_manifest_sha256", match=MatchValue(value=base_generation_manifest_sha256)), + FieldCondition(key="pr_generation_fingerprint", match=MatchValue(value=pr_generation_fingerprint)), + ] + if exact_binding else [] + ), + ], + must_not=[FieldCondition( + key=PR_OVERLAY_MANIFEST_PAYLOAD_KEY, + match=MatchValue(value=True), + )], ) direct_file_results = _fetch_direct_pr_file_chunks( @@ -214,7 +395,21 @@ def _query_pr_indexed_data( with_payload=True, with_vectors=False ) + if exact_binding: + for point in results: + payload = point.payload or {} + if any(payload.get(key) != value for key, value in ( + ("pr_source_revision", source_revision), + ("pr_base_revision", base_revision), + ("pr_base_generation_manifest_sha256", base_generation_manifest_sha256), + ("pr_generation_fingerprint", pr_generation_fingerprint), + )): + raise IncrementalIndexPreconditionError( + "PR point is outside the requested overlay generation" + ) accepted = query_service._accept_stored_points(results) + if not isinstance(accepted, list): + accepted = results formatted = _format_pr_results(accepted[:top_k]) return _merge_pr_results(direct_file_results, formatted) @@ -238,6 +433,8 @@ def _query_pr_indexed_data( return _merge_pr_results(direct_file_results, formatted) except Exception as e: + if exact_binding: + raise logger.warning(f"Error querying PR-indexed data: {e}") return [] @@ -350,8 +547,69 @@ def get_deterministic_context(request: DeterministicContextRequest): No language-specific parsing needed - tree-sitter already did it during indexing. Predictable: same input = same output. """ - _, query_service = _get_singletons() + index_manager, query_service = _get_singletons() try: + target_branch = request.branches[0] if request.branches else None + source_revision = _optional_string(request.source_revision) + base_revision = _optional_string(request.base_revision) + base_generation_manifest = _optional_string( + request.base_generation_manifest_sha256 + ) + pr_generation_fingerprint = _optional_string( + request.pr_generation_fingerprint + ) + pr_overlay_manifest = _optional_string( + request.pr_overlay_generation_manifest_sha256 + ) + collection_target = _optional_string(request.collection_target) + receipt = None + overlay_receipt = None + if base_revision and target_branch: + if len(request.branches) != 1: + raise IncrementalIndexPreconditionError( + "revision-bound deterministic context requires exactly one authoritative branch" + ) + receipt = require_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=target_branch, + revision=base_revision, + generation_manifest_sha256=base_generation_manifest, + collection_target=collection_target, + ) + exact_overlay_binding = all(( + request.pr_number, + source_revision, + base_revision, + base_generation_manifest, + pr_generation_fingerprint, + pr_overlay_manifest, + )) + if exact_overlay_binding: + overlay_receipt = read_pr_overlay_generation( + index_manager.qdrant_client, + receipt["_collection_target"], + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=target_branch, + base_branch=target_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + generation_fingerprint=pr_generation_fingerprint, + overlay_representation_fingerprint=( + index_manager.pr_overlay_representation_fingerprint + ), + expected_manifest_sha256=( + pr_overlay_manifest + ), + ) + if overlay_receipt is None: + raise IncrementalIndexPreconditionError( + "requested PR overlay generation is unavailable" + ) context = query_service.get_deterministic_context( workspace=request.workspace, project=request.project, @@ -360,9 +618,52 @@ def get_deterministic_context(request: DeterministicContextRequest): limit_per_file=request.limit_per_file or 10, pr_number=request.pr_number, pr_changed_files=request.pr_changed_files, - additional_identifiers=request.additional_identifiers + additional_identifiers=request.additional_identifiers, + expected_revisions=( + {target_branch: base_revision} + if base_revision and target_branch else None + ), + pr_source_revision=source_revision, + pr_base_revision=base_revision, + pr_base_generation_manifest_sha256=base_generation_manifest, + pr_generation_fingerprint=pr_generation_fingerprint, + collection_target=( + receipt["_collection_target"] if receipt else collection_target + ), ) + if receipt: + require_same_repository_generation( + index_manager, + workspace=request.workspace, + project=request.project, + branch=target_branch, + revision=base_revision, + receipt=receipt, + ) + if overlay_receipt: + second_overlay = read_pr_overlay_generation( + index_manager.qdrant_client, + receipt["_collection_target"], + workspace=request.workspace, + project=request.project, + pr_number=request.pr_number, + branch=target_branch, + base_branch=target_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=base_generation_manifest, + generation_fingerprint=pr_generation_fingerprint, + overlay_representation_fingerprint=( + index_manager.pr_overlay_representation_fingerprint + ), + ) + if second_overlay != overlay_receipt: + raise IncrementalIndexPreconditionError( + "PR overlay generation changed while context was retrieved" + ) return {"context": context} + except IncrementalIndexPreconditionError as e: + raise HTTPException(status_code=409, detail=str(e)) except Exception as e: logger.error(f"Error getting deterministic context: {e}") raise HTTPException(status_code=500, detail=str(e)) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py index 8e4305504..562d5b9e0 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/coordination.py @@ -140,7 +140,14 @@ def close(self) -> None: class ProjectMutationCoordinator: - """Serialize collection mutations for one workspace/project across workers.""" + """Serialize mutations that share one RAG publication resource. + + Legacy indexes share a project-wide collection. Exact generations have an + immutable collection target, while a published branch also has one mutable + human-readable head alias. ``publication_scope`` serializes only that + branch head, so main and develop can build concurrently without allowing + two generations of the same branch to race its current alias. + """ def __init__( self, @@ -166,9 +173,23 @@ def __init__( ) @staticmethod - def _project_key(workspace: str, project: str) -> str: + def _resource_key( + workspace: str, + project: str, + collection_target: Optional[str] = None, + publication_scope: Optional[str] = None, + ) -> str: + # Legacy indexing keeps the historical project-wide key because its + # branches share one physical alias. Exact branch generations supply + # their distinct collection target, so independent branch snapshots + # may proceed concurrently without weakening same-collection safety. + resource = ( + publication_scope + or collection_target + or "project-shared-collection" + ) digest = hashlib.sha256( - f"{workspace}\0{project}".encode("utf-8") + f"{workspace}\0{project}\0{resource}".encode("utf-8") ).hexdigest() return f"codecrow:rag:mutation:{digest}" @@ -178,6 +199,9 @@ def acquire( workspace: str, project: str, operation: str, + *, + collection_target: Optional[str] = None, + publication_scope: Optional[str] = None, ) -> Iterator[MutationLease]: token = uuid.uuid4().hex if not self.enabled or self._client is None: @@ -185,7 +209,12 @@ def acquire( yield lease return - key = self._project_key(workspace, project) + key = self._resource_key( + workspace, + project, + collection_target, + publication_scope, + ) operation_key = f"codecrow:rag:operation:{token}" deadline = time.monotonic() + self.acquire_timeout_seconds while True: @@ -221,7 +250,16 @@ def acquire( ) from exception if time.monotonic() >= deadline: raise MutationLeaseUnavailable( - f"another RAG mutation is active for {workspace}/{project}" + "another RAG mutation is active for " + f"{workspace}/{project}" + + ( + f" publication {publication_scope}" + if publication_scope + else ( + f" collection {collection_target}" + if collection_target else "" + ) + ) ) time.sleep(0.1) @@ -234,10 +272,11 @@ def acquire( ) lease.start_renewal() logger.info( - "Acquired RAG mutation lease operation=%s workspace=%s project=%s operation_id=%s", + "Acquired RAG mutation lease operation=%s workspace=%s project=%s collection=%s operation_id=%s", operation, workspace, project, + publication_scope or collection_target or "project-shared-collection", token, ) try: diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py new file mode 100644 index 000000000..75c5977b3 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/generation_manifest.py @@ -0,0 +1,379 @@ +"""Content-addressed membership manifests for repository index generations.""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +from llama_index.core.schema import TextNode +from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct + + +GENERATION_MANIFEST_PAYLOAD_KEY = "repository_generation_manifest" +GENERATION_MEMBER_DIGEST_PAYLOAD_KEY = "generation_member_sha256" +GENERATION_SCHEMA = "codecrow.repository-index-generation" +INDEX_SELECTION_POLICY_SCHEMA = "codecrow.repository-index-selection" +GENERATION_MANIFEST_PATH = ( + "__analysis_state__/repository-generation-manifest/000000.state" +) + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +class GenerationManifestError(RuntimeError): + """A repository generation is incomplete or its seal is inconsistent.""" + + +def _canonical_json(value: Any) -> str: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def _canonical_vector(value: Any) -> Any: + """Canonicalize cosine vectors as Qdrant stores them. + + Qdrant normalizes cosine vectors and persists float32 values. Rounding the + normalized representation avoids treating harmless transport precision as + a generation-integrity failure while still binding every vector value. + """ + if isinstance(value, Mapping): + return { + str(key): _canonical_vector(item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + } + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)): + values = list(value) + if values and all(isinstance(item, (int, float)) for item in values): + norm = math.sqrt(sum(float(item) ** 2 for item in values)) + if norm: + return [round(float(item) / norm, 7) for item in values] + return [0.0 for _ in values] + return [_canonical_vector(item) for item in values] + return value + + +def canonical_index_selection_policy( + include_patterns: Sequence[str] | None, + exclude_patterns: Sequence[str] | None, +) -> dict[str, Any]: + """Return the order-independent effective repository selection policy.""" + + for label, patterns in ( + ("include", include_patterns), + ("exclude", exclude_patterns), + ): + if patterns is not None and ( + isinstance(patterns, (str, bytes)) + or not all(isinstance(pattern, str) for pattern in patterns) + ): + raise GenerationManifestError( + f"repository index {label} patterns are invalid" + ) + return { + "schema": INDEX_SELECTION_POLICY_SCHEMA, + "includePatterns": sorted(set(include_patterns or ())), + "excludePatterns": sorted(set(exclude_patterns or ())), + } + + +def compute_index_selection_policy_sha256( + include_patterns: Sequence[str] | None, + exclude_patterns: Sequence[str] | None, +) -> str: + """Digest the canonical effective repository selection policy.""" + + policy = canonical_index_selection_policy( + include_patterns, + exclude_patterns, + ) + return hashlib.sha256(_canonical_json(policy).encode("utf-8")).hexdigest() + + +def compute_generation_member_digest( + point_id: object, + payload: Mapping[str, Any], + vector: object, +) -> str: + """Bind one persisted point's deterministic identity, payload and vector. + + ``indexed_at`` is operational metadata, not representation content. It is + excluded so rebuilding identical repository content can produce the same + generation identity. The digest field itself is also excluded to avoid a + recursive projection. + """ + content_payload = { + key: value + for key, value in payload.items() + if key not in { + GENERATION_MEMBER_DIGEST_PAYLOAD_KEY, + "indexed_at", + } + } + encoded = _canonical_json({ + "id": str(point_id), + "payload": content_payload, + "vector": _canonical_vector(vector), + }).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def compute_generation_members_digest( + members: Iterable[tuple[object, str]], +) -> str: + """Return the order-independent aggregate identity for generation members.""" + normalized = [] + seen_ids = set() + for point_id, member_digest in members: + normalized_id = str(point_id) + if normalized_id in seen_ids: + raise GenerationManifestError( + "repository generation contains duplicate point identities" + ) + if not isinstance(member_digest, str) or not _SHA256_RE.fullmatch( + member_digest + ): + raise GenerationManifestError( + "repository generation member is missing a valid content digest" + ) + seen_ids.add(normalized_id) + normalized.append((normalized_id, member_digest)) + + hasher = hashlib.sha256() + for point_id, member_digest in sorted(normalized): + encoded_id = point_id.encode("utf-8") + hasher.update(len(encoded_id).to_bytes(8, "big")) + hasher.update(encoded_id) + hasher.update(bytes.fromhex(member_digest)) + return hasher.hexdigest() + + +def verified_generation_member(point) -> tuple[object, str]: + """Recompute and verify one persisted generation member's full content.""" + payload = point.payload or {} + stored_digest = payload.get(GENERATION_MEMBER_DIGEST_PAYLOAD_KEY) + if not is_sha256_hex(stored_digest): + raise GenerationManifestError( + "repository generation member is missing a valid content digest" + ) + computed_digest = compute_generation_member_digest( + point.id, + payload, + point.vector, + ) + if computed_digest != stored_digest: + raise GenerationManifestError( + "repository generation member content digest does not match its " + "persisted payload and vector" + ) + return point.id, computed_digest + + +def _generation_filter(branch: str, commit: str) -> Filter: + return Filter( + must=[ + FieldCondition(key="branch", match=MatchValue(value=branch)), + FieldCondition(key="commit", match=MatchValue(value=commit)), + ], + must_not=[ + FieldCondition(key="pr", match=MatchValue(value=True)), + ], + ) + + +def collect_generation_members( + client, + collection_name: str, + branch: str, + commit: str, +) -> list[tuple[object, str]]: + """Read all unsealed members of one exact pending generation.""" + members = [] + offset = None + while True: + points, offset = client.scroll( + collection_name=collection_name, + scroll_filter=_generation_filter(branch, commit), + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + for point in points: + payload = point.payload or {} + if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: + raise GenerationManifestError( + "pending repository generation already contains a manifest" + ) + members.append(verified_generation_member(point)) + if offset is None: + break + return members + + +def seal_generation_members( + client, + collection_name: str, + branch: str, + commit: str, +) -> int: + """Persist member digests against Qdrant's actual stored vectors. + + Qdrant may normalize cosine vectors and reduce their precision. A digest + made from the embedding request is therefore not necessarily a digest of + the representation which will later be verified. Seal after every member + is stored, then collect/verify the same representation before publishing a + generation manifest. + """ + offset = None + sealed = 0 + while True: + points, offset = client.scroll( + collection_name=collection_name, + scroll_filter=_generation_filter(branch, commit), + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + replacements = [] + for point in points: + payload = dict(point.payload or {}) + if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: + raise GenerationManifestError( + "pending repository generation already contains a manifest" + ) + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = ( + compute_generation_member_digest(point.id, payload, point.vector) + ) + replacements.append(PointStruct( + id=point.id, + vector=point.vector, + payload=payload, + )) + if replacements: + client.upsert( + collection_name=collection_name, + points=replacements, + wait=True, + ) + sealed += len(replacements) + if offset is None: + break + return sealed + + +def generation_manifest_content( + *, + workspace: str, + project: str, + branch: str, + commit: str, + member_count: int, + members_sha256: str, + source_tree_sha256: str, + index_include_patterns: Sequence[str], + index_exclude_patterns: Sequence[str], + index_selection_policy_sha256: str, +) -> str: + """Serialize the immutable generation seal content.""" + selection_policy = canonical_index_selection_policy( + index_include_patterns, + index_exclude_patterns, + ) + return _canonical_json({ + "branch": branch, + "commit": commit, + "indexSelectionPolicy": selection_policy, + "indexSelectionPolicySha256": index_selection_policy_sha256, + "memberCount": member_count, + "membersSha256": members_sha256, + "project": project, + "schema": GENERATION_SCHEMA, + "sourceTreeSha256": source_tree_sha256, + "workspace": workspace, + }) + + +def build_generation_manifest_node( + *, + workspace: str, + project: str, + branch: str, + commit: str, + member_count: int, + members_sha256: str, + source_tree_sha256: str, + index_include_patterns: Sequence[str], + index_exclude_patterns: Sequence[str], + identity_metadata: Mapping[str, Any], +) -> TextNode: + """Build the single zero-vector state node that seals a generation.""" + if member_count < 1: + raise GenerationManifestError( + "repository generation cannot be sealed without members" + ) + if not _SHA256_RE.fullmatch(members_sha256): + raise GenerationManifestError( + "repository generation aggregate digest is invalid" + ) + if not _SHA256_RE.fullmatch(source_tree_sha256): + raise GenerationManifestError( + "repository generation source-tree digest is invalid" + ) + selection_policy = canonical_index_selection_policy( + index_include_patterns, + index_exclude_patterns, + ) + selection_policy_sha256 = compute_index_selection_policy_sha256( + selection_policy["includePatterns"], + selection_policy["excludePatterns"], + ) + content = generation_manifest_content( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=member_count, + members_sha256=members_sha256, + source_tree_sha256=source_tree_sha256, + index_include_patterns=selection_policy["includePatterns"], + index_exclude_patterns=selection_policy["excludePatterns"], + index_selection_policy_sha256=selection_policy_sha256, + ) + content_sha256 = hashlib.sha256(content.encode("utf-8")).hexdigest() + return TextNode( + text=content, + metadata={ + "workspace": workspace, + "project": project, + "branch": branch, + "commit": commit, + "path": GENERATION_MANIFEST_PATH, + "language": "repository-state", + "filetype": "state", + GENERATION_MANIFEST_PAYLOAD_KEY: True, + "generation_schema": GENERATION_SCHEMA, + "generation_member_count": member_count, + "generation_members_sha256": members_sha256, + "generation_manifest_sha256": content_sha256, + "source_tree_sha256": source_tree_sha256, + "index_include_patterns": selection_policy["includePatterns"], + "index_exclude_patterns": selection_policy["excludePatterns"], + "index_selection_policy_sha256": selection_policy_sha256, + **dict(identity_metadata), + }, + ) + + +def is_sha256_hex(value: object) -> bool: + """Return whether ``value`` is one canonical lower-case SHA-256 digest.""" + return isinstance(value, str) and _SHA256_RE.fullmatch(value) is not None diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/branch_manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/branch_manager.py index abd33ed74..f984a8be6 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/branch_manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/branch_manager.py @@ -114,10 +114,10 @@ def preserve_other_branch_points( offset = None total_yielded = 0 - + try: while True: - results = self.client.scroll( + points, next_offset = self.client.scroll( collection_name=collection_name, limit=batch_size, offset=offset, @@ -132,16 +132,15 @@ def preserve_other_branch_points( with_payload=True, with_vectors=True ) - points, next_offset = results - + if points: total_yielded += len(points) yield points - - if next_offset is None or len(points) < batch_size: + + if next_offset is None: break offset = next_offset - + logger.info(f"Streamed {total_yielded} points from other branches") except Exception as e: logger.warning(f"Could not read existing points: {e}") @@ -189,10 +188,10 @@ def stream_copy_points_to_collection( f"Re-embedding required for all branches." ) return 0 - except Exception as e: logger.warning(f"Could not verify collection dimensions: {e}") - # Continue anyway - will fail at upsert if dimensions don't match + # Optional preservation remains best effort. The target branch can + # still be published without copying unrelated legacy branches. total_copied = 0 for batch in self.preserve_other_branch_points(source_collection, exclude_branch, batch_size): diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py index 873ed0ff3..f2a43d84b 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/collection_manager.py @@ -9,7 +9,7 @@ import re import time import uuid -from typing import Callable, Optional, List +from typing import Callable, Mapping, Optional, List from qdrant_client import QdrantClient from qdrant_client.http.exceptions import UnexpectedResponse @@ -187,6 +187,30 @@ def resolve_alias(self, alias_name: str) -> Optional[str]: except Exception as e: logger.debug(f"Error resolving alias {alias_name}: {e}") return None + + def resolve_collection_target(self, collection_name: str) -> Optional[str]: + """Resolve an alias or direct collection without hiding backend errors. + + Mutation leases use this strict resolver so a transient alias lookup + failure cannot be mistaken for a direct collection. + """ + aliases = self.client.get_aliases().aliases + matching_aliases = [ + alias.collection_name + for alias in aliases + if alias.alias_name == collection_name + ] + if len(matching_aliases) > 1: + raise RuntimeError( + f"collection alias '{collection_name}' has multiple targets" + ) + if matching_aliases: + return matching_aliases[0] + + collections = self.client.get_collections().collections + if collection_name in {collection.name for collection in collections}: + return collection_name + return None def atomic_alias_swap( self, @@ -213,6 +237,63 @@ def atomic_alias_swap( change_aliases_operations=alias_operations ) logger.info(f"Alias swap completed: {alias_name} -> {new_collection}") + + def read_alias_targets(self, alias_names: List[str]) -> dict[str, Optional[str]]: + """Read several alias targets from one consistent Qdrant response.""" + requested = list(dict.fromkeys(name for name in alias_names if name)) + aliases = { + alias.alias_name: alias.collection_name + for alias in self.client.get_aliases().aliases + } + return {name: aliases.get(name) for name in requested} + + def atomic_assign_aliases( + self, + assignments: Mapping[str, Optional[str]], + ) -> None: + """Atomically point a set of aliases at already-validated collections. + + An immutable generation alias and its human-facing aliases must move in + the same Qdrant transaction. Reads that bind analysis to a historical + generation keep using the immutable alias; readable aliases are solely + the current branch and legacy-project pointers. + """ + desired = { + alias_name: collection_name + for alias_name, collection_name in assignments.items() + if alias_name + } + if not desired: + return + current = self.read_alias_targets(list(desired)) + operations = [] + for alias_name, collection_name in desired.items(): + if current.get(alias_name) == collection_name: + continue + if current.get(alias_name) is not None: + operations.append( + DeleteAliasOperation( + delete_alias=DeleteAlias(alias_name=alias_name) + ) + ) + if collection_name is not None: + operations.append( + CreateAliasOperation( + create_alias=CreateAlias( + alias_name=alias_name, + collection_name=collection_name, + ) + ) + ) + if not operations: + return + self.client.update_collection_aliases( + change_aliases_operations=operations + ) + logger.info( + "Atomically assigned Qdrant aliases: %s", + ", ".join(sorted(desired)), + ) def delete_alias(self, alias_name: str) -> bool: """Delete an alias.""" diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py index 856bad487..da808628d 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/indexer.py @@ -30,6 +30,13 @@ index_representation_fingerprint, observe_branch_representation, ) +from ..generation_manifest import ( + build_generation_manifest_node, + collect_generation_members, + compute_generation_members_digest, + seal_generation_members, +) +from ..source_tree import require_repository_source_tree_unchanged from .collection_manager import CollectionManager from .branch_manager import BranchManager from .point_operations import PointOperations @@ -48,6 +55,10 @@ def _plugin_identity_metadata( representation_fingerprint: Optional[str] = None, ): metadata = { + "plugin_ids": [], + "plugin_fingerprint": "sha256:" + "0" * 64, + "plugin_descriptor_fingerprint": "sha256:" + "0" * 64, + "plugin_implementation_fingerprint": "sha256:" + "0" * 64, INDEX_REPRESENTATION_PAYLOAD_KEY: ( representation_fingerprint or index_representation_fingerprint() @@ -482,6 +493,10 @@ def index_repository( preserve_other_branches: bool = False, include_patterns: Optional[List[str]] = None, exclude_patterns: Optional[List[str]] = None, + source_tree_sha256: Optional[str] = None, + source_tree=None, + seal_generation: bool = False, + publication_aliases: Optional[List[str]] = None, operation_id: Optional[str] = None, activation_guard: Optional[Callable[[], None]] = None, progress_callback: Optional[Callable[[dict], None]] = None, @@ -492,6 +507,7 @@ def report_progress( message: str, progress: Optional[int] = None, total: Optional[int] = None, + **details, ) -> None: if progress_callback is None: return @@ -500,6 +516,8 @@ def report_progress( event["progress"] = max(0, min(100, progress)) if total is not None: event["total"] = total + event.update({key: value for key, value in details.items() + if value is not None}) try: progress_callback(event) except Exception as exception: @@ -532,6 +550,13 @@ def report_progress( operation_id=operation_id, ) + activation_aliases = list(dict.fromkeys( + [alias_name, *(publication_aliases or [])] + )) + activation_alias_targets = self.collection_manager.read_alias_targets( + activation_aliases + ) + # Check existing collection and preserve other branch data using streaming old_alias_exists = self.collection_manager.alias_exists(alias_name) old_collection_exists = old_alias_exists or self.collection_manager.collection_exists(alias_name) @@ -549,7 +574,14 @@ def report_progress( # Get file list repository_file_list = list( - self.loader.iter_repository_files(repo_path_obj, include_patterns, exclude_patterns) + self.loader.iter_repository_files( + repo_path_obj, + include_patterns, + exclude_patterns, + expected_file_sha256=( + source_tree.file_sha256_by_path if source_tree else None + ), + ) ) logger.info( "Found %s repository files before plugin file policy for branch '%s'", @@ -673,6 +705,23 @@ def report_progress( skipped_file_paths: set[str] = set() preserved_point_count = 0 embedding_metrics = {"reused": 0, "embedded": 0} + estimated_chunks = None + + # Chunk totals are an estimate rather than an expensive second exact + # indexing pass. Progress remains useful even when estimation fails. + if progress_callback is not None: + try: + _, estimated_chunks = self.estimate_repository_size( + repo_path, include_patterns, exclude_patterns + ) + report_progress( + "estimating", + f"Estimated approximately {estimated_chunks} chunks", + 14, + estimatedChunks=estimated_chunks, + ) + except Exception as exception: + logger.warning("Could not estimate RAG progress chunks: %s", exception) try: # A main-only project must not carry stale non-target branches into @@ -700,6 +749,9 @@ def report_progress( f"Starting {total_batches} indexing batches", 18, total_batches, + totalBatches=total_batches, + indexedChunks=0, + estimatedChunks=estimated_chunks, ) # Architecture-only files still have to reach the repository @@ -715,6 +767,9 @@ def report_progress( documents = self.loader.load_file_batch( file_batch, repo_path_obj, workspace, project, branch, commit, strict=False, + expected_file_sha256=( + source_tree.file_sha256_by_path if source_tree else None + ), ) load_duration_ms = round( (time.perf_counter() - load_started) * 1000 @@ -829,6 +884,11 @@ def report_progress( round((time.perf_counter() - batch_started) * 1000), ) batch_progress = 18 + round(67 * batch_num / max(total_batches, 1)) + elapsed_ms = round((time.perf_counter() - operation_started) * 1000) + average_batch_ms = elapsed_ms / batch_num + estimated_remaining_ms = round( + average_batch_ms * max(total_batches - batch_num, 0) + ) report_progress( "indexing", ( @@ -838,6 +898,14 @@ def report_progress( ), batch_progress, total_batches, + indexedChunks=successful_chunks, + estimatedChunks=estimated_chunks, + completedBatches=batch_num, + totalBatches=total_batches, + batchDurationMs=round( + (time.perf_counter() - batch_started) * 1000 + ), + estimatedRemainingMs=estimated_remaining_ms, ) del documents @@ -949,6 +1017,73 @@ def report_progress( len(facts_nodes), ) + generation_manifest_sha256 = None + generation_manifest_points = 0 + if seal_generation: + report_progress( + "sealing", + "Sealing persisted vectors for generation integrity", + 91, + indexedChunks=successful_chunks, + estimatedChunks=estimated_chunks, + ) + seal_generation_members( + self.point_ops.client, + pending_collection_name, + branch, + commit, + ) + identity_metadata = _plugin_identity_metadata( + capabilities, + implementation_fingerprint, + self.representation_fingerprint, + ) + members = collect_generation_members( + self.point_ops.client, + pending_collection_name, + branch, + commit, + ) + if source_tree is not None: + require_repository_source_tree_unchanged( + repo_path_obj, + source_tree, + ) + if not source_tree_sha256: + raise RuntimeError( + "repository source-tree identity is required to seal an index generation" + ) + manifest = build_generation_manifest_node( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=len(members), + members_sha256=compute_generation_members_digest(members), + source_tree_sha256=source_tree_sha256, + index_include_patterns=include_patterns or (), + index_exclude_patterns=exclude_patterns or (), + identity_metadata=identity_metadata, + ) + manifest_success, manifest_failed = ( + self.point_ops.process_and_upsert_chunks( + [manifest], + pending_collection_name, + workspace, + project, + branch, + operation_id=operation_id, + ) + ) + if manifest_success != 1 or manifest_failed: + raise RuntimeError( + "repository generation manifest could not be persisted" + ) + generation_manifest_points = 1 + generation_manifest_sha256 = manifest.metadata[ + "generation_manifest_sha256" + ] + logger.info( f"Streaming indexing complete: {document_count} files, " f"{successful_chunks}/{chunk_count} chunks indexed " @@ -960,7 +1095,11 @@ def report_progress( report_progress("verifying", "Verifying the pending vector collection", 94) pending_info = self.point_ops.client.get_collection(pending_collection_name) actual_point_count = int(pending_info.points_count or 0) - expected_point_count = preserved_point_count + successful_chunks + expected_point_count = ( + preserved_point_count + + successful_chunks + + generation_manifest_points + ) if actual_point_count != expected_point_count: raise RuntimeError( "Pending collection point count is incomplete: " @@ -973,29 +1112,32 @@ def report_progress( branch, ) ) - if target_branch_point_count != successful_chunks: + expected_target_branch_points = ( + successful_chunks + generation_manifest_points + ) + if target_branch_point_count != expected_target_branch_points: raise RuntimeError( "Pending target-branch point count is incomplete: " - f"branch={branch}, expected={successful_chunks}, " + f"branch={branch}, expected={expected_target_branch_points}, " f"actual={target_branch_point_count}" ) if activation_guard is not None: activation_guard() - observed_target = self.collection_manager.resolve_alias(alias_name) - if old_alias_exists and observed_target != actual_old_collection: - raise RuntimeError( - "Active RAG collection changed before pending activation" - ) - if not old_alias_exists and observed_target is not None: + observed_targets = self.collection_manager.read_alias_targets( + activation_aliases + ) + if observed_targets != activation_alias_targets: raise RuntimeError( - "RAG alias was created concurrently before pending activation" + "Active RAG alias changed before pending activation" ) activation_started = time.perf_counter() report_progress("activating", "Activating the completed vector collection", 97) - old_target = self._perform_atomic_swap( - alias_name, pending_collection_name, old_alias_exists + old_targets = self._perform_atomic_swap( + alias_name, + pending_collection_name, + activation_aliases, ) logger.info( "RAG pending collection activated operation_id=%s collection=%s " @@ -1015,10 +1157,18 @@ def report_progress( successful_chunks, ) except Exception: - self._rollback_atomic_swap(alias_name, old_target) + self._rollback_atomic_swap(old_targets) raise - if old_target and old_target != pending_collection_name: + old_target = old_targets.get(alias_name) + # Exact aliases identify prior sealed revisions that can still be + # selected by PR analysis. Their registry retention owns physical + # cleanup; a replacement build must not delete them here. + if ( + not seal_generation + and old_target + and old_target != pending_collection_name + ): self.collection_manager.delete_collection(old_target) except Exception as e: @@ -1048,6 +1198,11 @@ def report_progress( f"Indexed {document_count} files into {successful_chunks} chunks", 100, document_count, + indexedChunks=successful_chunks, + estimatedChunks=successful_chunks, + completedBatches=total_batches, + totalBatches=total_batches, + estimatedRemainingMs=0, ) return IndexStats( namespace=namespace, @@ -1058,33 +1213,29 @@ def report_progress( last_updated=datetime.now(timezone.utc).isoformat(), workspace=workspace, project=project, - branch=branch + branch=branch, + generation_manifest_sha256=generation_manifest_sha256, + source_tree_sha256=source_tree_sha256, + collection_target=alias_name, ) def _perform_atomic_swap( - self, - alias_name: str, - pending_collection_name: str, - old_alias_exists: bool - ) -> Optional[str]: - """Activate a complete pending collection and retain its rollback target.""" + self, + alias_name: str, + pending_collection_name: str, + activation_aliases: List[str], + ) -> dict[str, Optional[str]]: + """Activate a complete generation and its readable aliases together.""" logger.info("Performing atomic alias swap...") - - old_target = self.collection_manager.resolve_alias(alias_name) if old_alias_exists else None - self.collection_manager.atomic_alias_swap( - alias_name, - pending_collection_name, - old_alias_exists, + old_targets = self.collection_manager.read_alias_targets(activation_aliases) + self.collection_manager.atomic_assign_aliases( + {alias: pending_collection_name for alias in activation_aliases} ) - return old_target + return old_targets - def _rollback_atomic_swap(self, alias_name: str, old_target: Optional[str]) -> None: - """Restore the previously active collection after metadata publication fails.""" - if old_target: - self.collection_manager.atomic_alias_swap(alias_name, old_target, True) - return - if not self.collection_manager.delete_alias(alias_name): - logger.critical("Failed to remove newly activated alias %s during rollback", alias_name) + def _rollback_atomic_swap(self, old_targets: dict[str, Optional[str]]) -> None: + """Restore every alias changed during a failed metadata publication.""" + self.collection_manager.atomic_assign_aliases(old_targets) class FileOperations: @@ -1217,6 +1368,7 @@ def _replace_points( mutation_guard: Optional[Callable[[], None]] = None, ) -> int: """Upsert a prepared generation, delete stale IDs, and roll back on error.""" + old_records = list(old_records) old_points = {str(record.id): record for record in old_records} chunk_data = self.point_ops.prepare_chunks_for_embedding( nodes, @@ -1239,6 +1391,25 @@ def _replace_points( embedding_metrics["reused"], embedding_metrics["embedded"], ) + return self._replace_prepared_points( + new_points, + old_records, + collection_name, + mutation_guard, + allow_skipped=True, + ) + + def _replace_prepared_points( + self, + new_points, + old_records, + collection_name: str, + mutation_guard: Optional[Callable[[], None]] = None, + *, + allow_skipped: bool, + ) -> int: + """Publish prepared points with rollback and explicit skip semantics.""" + old_points = {str(record.id): record for record in old_records} new_ids = {str(point.id) for point in new_points} old_ids = set(old_points) new_only_ids = [ @@ -1264,6 +1435,15 @@ def _replace_points( str(point.id) for point in write_result.skipped_points } if skipped_ids: + if not allow_skipped: + self._restore_old_points( + collection_name, + old_points, + new_only_ids, + ) + raise RuntimeError( + "an exact index generation contains rejected points" + ) logger.warning( "Quarantining %s rejected points during incremental replacement", len(skipped_ids), @@ -1287,6 +1467,105 @@ def _replace_points( raise return write_result.successful + def replace_pr_overlay_generation( + self, + nodes, + old_records, + collection_name: str, + workspace: str, + project: str, + point_id_branch: str, + mutation_guard: Optional[Callable[[], None]] = None, + *, + pr_number: int, + branch: str, + base_branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, + identity_metadata, + ) -> tuple[int, dict]: + """Embed all PR members, seal them, and publish one complete set.""" + from ..pr_overlay_manifest import build_pr_overlay_manifest_node + + old_records = list(old_records) + old_points = {str(record.id): record for record in old_records} + chunk_data = self.point_ops.prepare_chunks_for_embedding( + nodes, workspace, project, point_id_branch + ) + new_points = self.point_ops.embed_and_create_points( + chunk_data, + reuse_records=old_points.values(), + ) + new_ids = [point.id for point in new_points] + if mutation_guard is not None: + mutation_guard() + successful, failed = self.point_ops.upsert_points( + collection_name, new_points + ) + if failed or successful != len(new_points): + self._delete_point_ids(collection_name, new_ids) + raise RuntimeError("PR overlay member write was incomplete") + try: + members = self.point_ops._seal_persisted_point_digests( + collection_name, new_points + ) + manifest_node, receipt = build_pr_overlay_manifest_node( + workspace=workspace, + project=project, + pr_number=pr_number, + branch=branch, + base_branch=base_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_manifest_sha256 + ), + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + members=members, + identity_metadata=identity_metadata, + ) + manifest_data = self.point_ops.prepare_chunks_for_embedding( + [manifest_node], workspace, project, point_id_branch + ) + manifest_points = self.point_ops.embed_and_create_points( + manifest_data + ) + manifest_success, manifest_failed = self.point_ops.upsert_points( + collection_name, manifest_points + ) + if manifest_success != 1 or manifest_failed: + raise RuntimeError("PR overlay manifest write was incomplete") + except Exception: + self._delete_point_ids( + collection_name, + [*new_ids, *(point.id for point in locals().get("manifest_points", []))], + ) + raise + + active_ids = {str(point.id) for point in (*new_points, *manifest_points)} + stale_ids = [ + record.id for point_id, record in old_points.items() + if point_id not in active_ids + ] + try: + if mutation_guard is not None: + mutation_guard() + self._delete_point_ids(collection_name, stale_ids) + except Exception: + self._restore_old_points( + collection_name, + old_points, + [*new_ids, *(point.id for point in manifest_points)], + ) + raise + return successful, receipt + def _apply_change_set( self, updated_file_paths: List[str], diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py index 0a5ae6dea..c769abeba 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py @@ -6,10 +6,14 @@ import logging import os +import hashlib +import json +import re from typing import Callable, Optional, List from llama_index.core import Settings from qdrant_client import QdrantClient +from qdrant_client.models import FieldCondition, Filter, MatchValue, PointStruct from ...models.config import RAGConfig, IndexStats from ...utils.utils import make_namespace, make_project_namespace @@ -24,6 +28,20 @@ from ..pr_overlay_representation import ( pr_overlay_representation_fingerprint, ) +from ..generation_manifest import ( + GENERATION_MANIFEST_PAYLOAD_KEY, + GENERATION_MEMBER_DIGEST_PAYLOAD_KEY, + build_generation_manifest_node, + collect_generation_members, + compute_generation_member_digest, + compute_generation_members_digest, +) +from .. import revision_preflight +from ..repository_overlay import IncrementalIndexPreconditionError +from ..source_tree import ( + compute_repository_source_tree_sha256, + verify_repository_source_tree, +) from .collection_manager import CollectionManager from .branch_manager import BranchManager @@ -34,6 +52,13 @@ logger = logging.getLogger(__name__) +def read_repository_revision_preflight(*args, **kwargs): + """Patchable module boundary around strict generation verification.""" + return revision_preflight.read_repository_revision_preflight( + *args, **kwargs + ) + + def _config_int(config, name: str, default: int) -> int: value = getattr(config, name, default) if isinstance(value, bool) or not isinstance(value, (int, str)): @@ -189,6 +214,60 @@ def _get_project_collection_name(self, workspace: str, project: str) -> str: namespace = make_project_namespace(workspace, project) return f"{self.config.qdrant_collection_prefix}_{namespace}" + def _get_branch_operator_alias( + self, + workspace: str, + project: str, + branch: str, + ) -> str: + """Return a readable, collision-safe alias for one branch's head. + + Exact review context always uses the immutable generation target from + the Java registry. This alias is an operator and legacy-integration + pointer to the current published head of one branch. + """ + legacy_alias = self._get_project_collection_name(workspace, project) + raw_branch = branch.strip() + readable = re.sub(r"[^a-zA-Z0-9_-]", "_", raw_branch).lower() + if not readable: + readable = "branch" + if readable != raw_branch.lower() or len(readable) > 64: + readable = ( + f"{readable[:64]}_" + f"{hashlib.sha256(raw_branch.encode('utf-8')).hexdigest()[:10]}" + ) + return f"{legacy_alias}__{readable}" + + def _publication_aliases( + self, + workspace: str, + project: str, + branch: str, + publish_branch_alias: bool, + publish_legacy_project_alias: bool, + ) -> List[str]: + aliases: List[str] = [] + if publish_branch_alias: + aliases.append(self._get_branch_operator_alias(workspace, project, branch)) + if publish_legacy_project_alias: + aliases.append(self._get_project_collection_name(workspace, project)) + return list(dict.fromkeys(aliases)) + + @staticmethod + def _publication_scope( + branch: str, + publication_aliases: List[str], + ) -> Optional[str]: + """Return the mutable resource shared by one branch head. + + Exact target aliases are immutable and may be built independently. + Once a build publishes a readable branch pointer, later builds of the + same branch must serialize their final alias activation. Other branch + scopes remain independent, so retained branches still index in + parallel. + """ + return f"branch-head:{branch}" if publication_aliases else None + # Repository indexing def estimate_repository_size( @@ -210,14 +289,38 @@ def index_repository( preserve_other_branches: bool = False, include_patterns: Optional[List[str]] = None, exclude_patterns: Optional[List[str]] = None, + source_tree_sha256: Optional[str] = None, + collection_target: Optional[str] = None, + publish_branch_alias: bool = False, + publish_legacy_project_alias: bool = False, progress_callback: Optional[Callable[[dict], None]] = None, ) -> IndexStats: """Index entire repository for a branch using atomic swap strategy.""" - alias_name = self._get_project_collection_name(workspace, project) + alias_name = collection_target or self._get_project_collection_name( + workspace, project + ) + expected_source_tree = ( + source_tree_sha256 + or compute_repository_source_tree_sha256(repo_path) + ) + source_tree = verify_repository_source_tree( + repo_path, + commit, + expected_source_tree, + ) + publication_aliases = self._publication_aliases( + workspace, + project, + branch, + publish_branch_alias, + publish_legacy_project_alias, + ) with self._mutation_coordinator.acquire( workspace, project, "full-index", + collection_target=collection_target, + publication_scope=self._publication_scope(branch, publication_aliases), ) as lease: return self._indexer.index_repository( repo_path=repo_path, @@ -229,11 +332,108 @@ def index_repository( preserve_other_branches=preserve_other_branches, include_patterns=include_patterns, exclude_patterns=exclude_patterns, + source_tree_sha256=source_tree.tree_sha256, + source_tree=source_tree, + seal_generation=collection_target is not None, + publication_aliases=publication_aliases, operation_id=lease.token, activation_guard=lease.assert_owned, progress_callback=progress_callback, ) + def get_revision_preflight( + self, + workspace: str, + project: str, + branch: str, + commit: str, + *, + collection_target: Optional[str] = None, + ): + """Verify one immutable revision in an explicitly selected target.""" + target = collection_target or self._get_project_collection_name( + workspace, project + ) + physical = self._collection_manager.resolve_collection_target(target) + if physical is None: + return None + result = read_repository_revision_preflight( + self.qdrant_client, + physical, + branch, + commit, + ) + if result is None: + return None + if ( + result.get("workspace") != workspace + or result.get("project") != project + ): + raise IncrementalIndexPreconditionError( + "repository generation coordinates do not match the requested tenant" + ) + return result + + def publish_generation_aliases( + self, + workspace: str, + project: str, + branch: str, + commit: str, + collection_target: str, + publish_branch_alias: bool = True, + publish_legacy_project_alias: bool = False, + ) -> List[str]: + """Repair operator aliases after proving one sealed generation identity. + + Normal indexing publishes these pointers atomically with its immutable + target alias. This idempotent path exists only for generations created + before that contract was introduced or when Qdrant has been restored. + """ + aliases = self._publication_aliases( + workspace, + project, + branch, + publish_branch_alias, + publish_legacy_project_alias, + ) + if not aliases: + return [] + with self._mutation_coordinator.acquire( + workspace, + project, + "publish-generation-aliases", + collection_target=collection_target, + publication_scope=self._publication_scope(branch, aliases), + ) as lease: + physical = self._collection_manager.resolve_collection_target( + collection_target + ) + if physical is None: + raise IncrementalIndexPreconditionError( + "repository generation is unavailable for alias publication" + ) + receipt = read_repository_revision_preflight( + self.qdrant_client, + physical, + branch, + commit, + ) + if receipt is None or any(receipt.get(key) != value for key, value in ( + ("workspace", workspace), + ("project", project), + ("branch", branch), + ("commit", commit), + )): + raise IncrementalIndexPreconditionError( + "repository generation does not match the requested alias coordinates" + ) + lease.assert_owned() + self._collection_manager.atomic_assign_aliases( + {alias: physical for alias in aliases} + ) + return aliases + # File operations def update_files( @@ -317,10 +517,327 @@ def apply_changes( mutation_guard=lease.assert_owned, ) + def _rebind_repository_facts_revision( + self, + collection_name: str, + branch: str, + commit: str, + ) -> None: + """Rebind the copied neutral inventory before applying its delta.""" + points = [] + offset = None + while True: + batch, offset = self.qdrant_client.scroll( + collection_name=collection_name, + scroll_filter=Filter(must=[ + FieldCondition(key="branch", match=MatchValue(value=branch)), + FieldCondition( + key="repository_facts_state", + match=MatchValue(value=True), + ), + ]), + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + points.extend(batch) + if offset is None: + break + if not points: + raise IncrementalIndexPreconditionError( + "source generation has no repository detection facts" + ) + ordered = sorted( + points, key=lambda point: (point.payload or {}).get("facts_part", -1) + ) + content = "".join((point.payload or {}).get("text", "") for point in ordered) + decoded = json.loads(content) + decoded["revision"] = commit + rebound = json.dumps( + decoded, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + parts = [ + rebound[index:index + 400_000] + for index in range(0, len(rebound), 400_000) + ] + if len(parts) != len(ordered): + raise IncrementalIndexPreconditionError( + "repository facts cannot be rebound incrementally; fully reindex" + ) + digest = hashlib.sha256(rebound.encode("utf-8")).hexdigest() + replacements = [] + for index, (point, text_part) in enumerate(zip(ordered, parts)): + payload = dict(point.payload or {}) + payload.update({ + "commit": commit, + "text": text_part, + "facts_part": index, + "facts_parts": len(parts), + "facts_content_sha256": digest, + }) + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = ( + compute_generation_member_digest(point.id, payload, point.vector) + ) + replacements.append(PointStruct( + id=point.id, vector=point.vector, payload=payload + )) + self.qdrant_client.upsert( + collection_name=collection_name, + points=replacements, + wait=True, + ) + + def advance_generation( + self, + source_collection_target: str, + target_collection_target: str, + source_commit: str, + source_tree_sha256: str, + updated_file_paths: List[str], + deleted_file_paths: List[str], + repo_base: Optional[str], + workspace: str, + project: str, + branch: str, + commit: str, + publish_branch_alias: bool = False, + publish_legacy_project_alias: bool = False, + ) -> IndexStats: + """Copy, mutate, seal, and atomically publish one exact generation.""" + publication_aliases = self._publication_aliases( + workspace, + project, + branch, + publish_branch_alias, + publish_legacy_project_alias, + ) + activation_aliases = list(dict.fromkeys( + [target_collection_target, *publication_aliases] + )) + with self._mutation_coordinator.acquire( + workspace, + project, + "advance-generation", + collection_target=target_collection_target, + publication_scope=self._publication_scope( + branch, + publication_aliases, + ), + ) as lease: + source_physical = self._collection_manager.resolve_collection_target( + source_collection_target + ) + if source_physical is None: + raise IncrementalIndexPreconditionError( + "source repository generation is unavailable" + ) + source_receipt = read_repository_revision_preflight( + self.qdrant_client, + source_physical, + branch, + source_commit, + ) + if source_receipt is None: + raise IncrementalIndexPreconditionError( + "source repository generation is not sealed" + ) + + target_physical = self._collection_manager.resolve_collection_target( + target_collection_target + ) + if target_physical is not None: + existing = read_repository_revision_preflight( + self.qdrant_client, + target_physical, + branch, + commit, + ) + if existing is None: + raise IncrementalIndexPreconditionError( + "target collection exists without the requested sealed generation" + ) + stats = self._stats_manager.get_branch_stats( + workspace, project, branch, target_physical + ) + if publication_aliases: + self._collection_manager.atomic_assign_aliases( + {alias: target_physical for alias in publication_aliases} + ) + if hasattr(stats, "model_copy"): + return stats.model_copy(update={ + "generation_manifest_sha256": existing[ + "generation_manifest_sha256" + ], + "source_tree_sha256": existing["source_tree_sha256"], + "collection_target": target_collection_target, + }) + return stats + + if repo_base is None: + raise IncrementalIndexPreconditionError( + "target repository snapshot is required for generation advance" + ) + observed_source_tree_sha256 = ( + compute_repository_source_tree_sha256(repo_base) + ) + if observed_source_tree_sha256 != source_tree_sha256: + raise IncrementalIndexPreconditionError( + "target repository source tree does not match its attestation" + ) + + pending = self._collection_manager.create_pending_collection( + target_collection_target, + operation_id=lease.token, + ) + activation_alias_targets = self._collection_manager.read_alias_targets( + activation_aliases + ) + activated = False + try: + offset = None + copied = 0 + while True: + points, offset = self.qdrant_client.scroll( + collection_name=source_physical, + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + batch = [] + for point in points: + payload = dict(point.payload or {}) + if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: + continue + if any(payload.get(key) != value for key, value in ( + ("workspace", workspace), + ("project", project), + ("branch", branch), + ("commit", source_commit), + )): + raise IncrementalIndexPreconditionError( + "source generation contains points outside its tenant or revision" + ) + payload["commit"] = commit + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = ( + compute_generation_member_digest( + point.id, payload, point.vector + ) + ) + batch.append(PointStruct( + id=point.id, + vector=point.vector, + payload=payload, + )) + if batch: + self.qdrant_client.upsert( + collection_name=pending, + points=batch, + wait=True, + ) + copied += len(batch) + if offset is None: + break + if copied < 1: + raise IncrementalIndexPreconditionError( + "source generation has no repository members" + ) + + self._rebind_repository_facts_revision( + pending, branch, commit + ) + + self._file_ops.apply_changes( + updated_file_paths=updated_file_paths, + deleted_file_paths=deleted_file_paths, + repo_base=repo_base, + workspace=workspace, + project=project, + branch=branch, + commit=commit, + collection_name=pending, + mutation_guard=lease.assert_owned, + ) + + members = collect_generation_members( + self.qdrant_client, pending, branch, commit + ) + identity = { + "plugin_ids": source_receipt["plugin_ids"], + "plugin_fingerprint": source_receipt["plugin_fingerprint"], + "plugin_descriptor_fingerprint": source_receipt[ + "plugin_descriptor_fingerprint" + ], + "plugin_implementation_fingerprint": source_receipt[ + "plugin_implementation_fingerprint" + ], + "index_representation_fingerprint": source_receipt[ + "index_representation_fingerprint" + ], + } + manifest = build_generation_manifest_node( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=len(members), + members_sha256=compute_generation_members_digest(members), + source_tree_sha256=source_tree_sha256, + index_include_patterns=source_receipt[ + "index_include_patterns" + ], + index_exclude_patterns=source_receipt[ + "index_exclude_patterns" + ], + identity_metadata=identity, + ) + success, failed = self._point_ops.process_and_upsert_chunks( + [manifest], pending, workspace, project, branch + ) + if success != 1 or failed: + raise RuntimeError("target generation manifest was not persisted") + lease.assert_owned() + if self._collection_manager.read_alias_targets( + activation_aliases) != activation_alias_targets: + raise RuntimeError( + "Active RAG alias changed before pending activation" + ) + self._collection_manager.atomic_assign_aliases( + {alias: pending for alias in activation_aliases} + ) + activated = True + stats = self._stats_manager.get_branch_stats( + workspace, project, branch, pending + ) + return stats.model_copy(update={ + "generation_manifest_sha256": manifest.metadata[ + "generation_manifest_sha256" + ], + "source_tree_sha256": source_tree_sha256, + "collection_target": target_collection_target, + }) + finally: + if not activated: + self._collection_manager.delete_collection(pending) + # Branch operations - def delete_branch(self, workspace: str, project: str, branch: str) -> bool: + def delete_branch( + self, + workspace: str, + project: str, + branch: str, + collection_target: Optional[str] = None, + ) -> bool: """Delete all points for a specific branch from the project collection.""" + if collection_target: + return self.delete_collection_target( + workspace, project, branch, collection_target + ) with self._mutation_coordinator.acquire( workspace, project, @@ -335,6 +852,55 @@ def delete_branch(self, workspace: str, project: str, branch: str) -> bool: lease.assert_owned() return self._branch_manager.delete_branch_points(collection_name, branch) + def delete_collection_target( + self, + workspace: str, + project: str, + branch: str, + collection_target: str, + ) -> bool: + """Delete one exact generation after proving its tenant ownership.""" + with self._mutation_coordinator.acquire( + workspace, project, "delete-generation" + ) as lease: + physical = self._collection_manager.resolve_collection_target( + collection_target + ) + if physical is None: + return False + offset = None + point_count = 0 + while True: + points, offset = self.qdrant_client.scroll( + collection_name=physical, + limit=256, + offset=offset, + with_payload=["workspace", "project", "branch"], + with_vectors=False, + ) + point_count += len(points) + for point in points: + payload = point.payload or {} + if any(payload.get(key) != value for key, value in ( + ("workspace", workspace), + ("project", project), + ("branch", branch), + )): + raise IncrementalIndexPreconditionError( + "collection target does not belong to the requested tenant branch" + ) + if offset is None: + break + if point_count == 0: + raise IncrementalIndexPreconditionError( + "empty collection target cannot be ownership-verified" + ) + lease.assert_owned() + if self._collection_manager.alias_exists(collection_target): + if not self._collection_manager.delete_alias(collection_target): + return False + return self._collection_manager.delete_collection(physical) + def get_branch_point_count(self, workspace: str, project: str, branch: str) -> int: """Get the number of points for a specific branch.""" collection_name = self._get_project_collection_name(workspace, project) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py index 547807b4f..e7640a49f 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/point_operations.py @@ -19,6 +19,11 @@ from qdrant_client import QdrantClient from qdrant_client.models import PointStruct +from ..generation_manifest import ( + GENERATION_MEMBER_DIGEST_PAYLOAD_KEY, + compute_generation_member_digest, +) + logger = logging.getLogger(__name__) EMBEDDING_INPUT_HASH_PAYLOAD_KEY = "embedding_input_sha256" @@ -108,6 +113,8 @@ def _is_architecture_chunk(chunk: TextNode) -> bool: or chunk.metadata.get("architecture_source") or chunk.metadata.get("repository_snapshot") or chunk.metadata.get("repository_facts_state") + or chunk.metadata.get("repository_generation_manifest") + or chunk.metadata.get("pr_overlay_generation_manifest") ) @staticmethod @@ -196,7 +203,7 @@ def _reusable_vectors( continue reusable[point_id] = vector return reusable - + @staticmethod def generate_point_id( workspace: str, @@ -297,7 +304,7 @@ def embed_and_create_points( metrics["embedded"] = metrics.get("embedded", 0) + len( embedded_by_id ) - + # Build points with embeddings points = [] for point_id, chunk in chunk_data: @@ -330,6 +337,9 @@ def embed_and_create_points( or chunk.metadata.get("repository_facts_state") ): payload["_node_content"] = chunk.text + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = ( + compute_generation_member_digest(point_id, payload, embedding) + ) points.append(PointStruct( id=point_id, vector=embedding, @@ -351,6 +361,44 @@ def upsert_points( result = self.upsert_points_detailed(collection_name, points) return result.successful, result.failed + def _seal_persisted_point_digests( + self, + collection_name: str, + points: List[PointStruct], + ) -> list[tuple[object, str]]: + """Bind digests to the vectors exactly as persisted by Qdrant.""" + if not points: + return [] + expected = {str(point.id): point for point in points} + records = self.client.retrieve( + collection_name=collection_name, + ids=[point.id for point in points], + with_payload=True, + with_vectors=True, + ) + if {str(record.id) for record in records} != set(expected): + raise RuntimeError("persisted point set is incomplete before sealing") + sealed = [] + replacements = [] + for record in records: + payload = dict(record.payload or {}) + digest = compute_generation_member_digest( + record.id, payload, record.vector + ) + payload[GENERATION_MEMBER_DIGEST_PAYLOAD_KEY] = digest + replacements.append(PointStruct( + id=record.id, + vector=record.vector, + payload=payload, + )) + sealed.append((record.id, digest)) + self.client.upsert( + collection_name=collection_name, + points=replacements, + wait=True, + ) + return sealed + def upsert_points_detailed( self, collection_name: str, diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_representation.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_representation.py index 45205180d..612915cd7 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_representation.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_representation.py @@ -16,6 +16,10 @@ INDEX_REPRESENTATION_PAYLOAD_KEY = "index_representation_fingerprint" + +class IndexCompatibilityError(RuntimeError): + """Deprecated compatibility name retained for older internal callers.""" + # These inputs can change persistent target-branch point text, metadata, or # vectors. PR-only request/overlay code is intentionally excluded so a PR # orchestration fix cannot force every repository embedding to be rebuilt. diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/loader.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/loader.py index d781bc0bb..9075d3544 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/loader.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/loader.py @@ -1,11 +1,16 @@ from pathlib import Path -from typing import List, Optional, Generator +from typing import List, Optional, Generator, Mapping import logging import re from llama_index.core.schema import Document -from ..utils.utils import detect_language_from_path, should_exclude_file, should_include_file, is_binary_file, clean_archive_path +from ..utils.utils import detect_language_from_path, should_exclude_file, should_include_file, clean_archive_path from ..models.config import RAGConfig +from .source_tree import ( + RepositorySourceTreeError, + iter_repository_regular_file_paths, + read_repository_file_bytes, +) logger = logging.getLogger(__name__) @@ -36,6 +41,15 @@ def _is_generated_asset(filename: str) -> bool: return has_letter and has_digit +def _decode_text(content: bytes) -> str | None: + if b"\0" in content: + return None + try: + return content.decode("utf-8") + except UnicodeDecodeError: + return None + + class DocumentLoader: """Load repository files as documents""" @@ -46,7 +60,8 @@ def iter_repository_files( self, repo_path: Path, extra_include_patterns: Optional[List[str]] = None, - extra_exclude_patterns: Optional[List[str]] = None + extra_exclude_patterns: Optional[List[str]] = None, + expected_file_sha256: Optional[Mapping[str, str]] = None, ) -> Generator[Path, None, None]: """Iterate over repository files without loading them into memory. @@ -78,15 +93,20 @@ def iter_repository_files( # Include patterns (project-specific only, no defaults) include_patterns = extra_include_patterns if extra_include_patterns else [] + candidates = ( + (repo_path / Path(path) for path in sorted(expected_file_sha256)) + if expected_file_sha256 is not None + else ( + repo_path / path + for path in iter_repository_regular_file_paths(repo_path) + ) + ) total_entries = 0 yielded_count = 0 - for file_path in repo_path.rglob("*"): + for file_path in candidates: total_entries += 1 - if not file_path.is_file(): - continue - relative_path = file_path.relative_to(repo_path) - relative_path_str = str(relative_path) + relative_path_str = relative_path.as_posix() # Step 1: Apply inclusion filter first # If include patterns are specified, only files matching at least one pattern pass @@ -97,10 +117,30 @@ def iter_repository_files( if should_exclude_file(relative_path_str, exclude_patterns): continue - if file_path.stat().st_size > self.config.max_file_size_bytes: + expected_digest = ( + expected_file_sha256.get(relative_path_str) + if expected_file_sha256 is not None + else None + ) + try: + content = read_repository_file_bytes( + repo_path, + relative_path, + expected_sha256=expected_digest, + ) + except RepositorySourceTreeError: + if expected_file_sha256 is not None: + raise + logger.warning( + "Cannot safely inspect repository file, skipping: %s", + relative_path_str, + ) continue - if is_binary_file(file_path): + if len(content) > self.config.max_file_size_bytes: + continue + + if _decode_text(content) is None: continue # Skip build-tool-generated assets with content hashes @@ -121,6 +161,7 @@ def load_file_batch( branch: str, commit: str, strict: bool = False, + expected_file_sha256: Optional[Mapping[str, str]] = None, ) -> List[Document]: """Load a batch of files as Documents. @@ -149,7 +190,22 @@ def load_file_batch( continue try: - text = full_path.read_text(encoding="utf-8") + expected_digest = None + if expected_file_sha256 is not None: + expected_digest = expected_file_sha256.get( + Path(relative_path).as_posix() + ) + if expected_digest is None: + raise RepositorySourceTreeError( + "repository source file was not present in the " + f"attested tree: {relative_path_str}" + ) + content = read_repository_file_bytes( + repo_base, + relative_path, + expected_sha256=expected_digest, + ) + text = content.decode("utf-8") if not text or not text.strip(): continue @@ -209,85 +265,20 @@ def load_from_directory( commit: Commit hash extra_exclude_patterns: Additional patterns to exclude (from project config) """ - documents = [] - - if not repo_path.exists(): - logger.error(f"Repository path does not exist: {repo_path}") - return documents - - # Combine default exclude patterns with project-specific ones - exclude_patterns = list(self.config.excluded_patterns) - if extra_exclude_patterns: - exclude_patterns.extend(extra_exclude_patterns) - logger.info(f"Using {len(extra_exclude_patterns)} additional exclude patterns from project config: {extra_exclude_patterns}") - - excluded_count = 0 - for file_path in repo_path.rglob("*"): - if not file_path.is_file(): - continue - - relative_path = str(file_path.relative_to(repo_path)) - - if should_exclude_file(relative_path, exclude_patterns): - logger.debug(f"Excluding file: {relative_path}") - excluded_count += 1 - continue - - if file_path.stat().st_size > self.config.max_file_size_bytes: - logger.warning(f"File too large, skipping: {relative_path}") - continue - - if is_binary_file(file_path): - logger.debug(f"Binary file, skipping: {relative_path}") - continue - - if _is_generated_asset(file_path.name): - logger.debug(f"Generated asset, skipping: {relative_path}") - excluded_count += 1 - continue - - try: - text = file_path.read_text(encoding="utf-8") - - # Skip empty files - if not text or not text.strip(): - logger.debug(f"Empty file, skipping: {relative_path}") - continue - - except UnicodeDecodeError: - logger.warning(f"Cannot decode file, skipping: {relative_path}") - continue - except Exception as e: - logger.error(f"Error reading file {relative_path}: {e}") - continue - - language = detect_language_from_path(str(file_path)) - filetype = file_path.suffix.lstrip('.') - - # Clean archive root prefix from path - clean_path = clean_archive_path(relative_path) - - metadata = { - "workspace": workspace, - "project": project, - "branch": branch, - "path": clean_path, - "commit": commit, - "language": language, - "filetype": filetype, - } - - doc = Document( - text=text, - metadata=metadata - # Don't set id_ - let LlamaIndex/Qdrant generate it automatically + file_paths = list( + self.iter_repository_files( + repo_path, + extra_exclude_patterns=extra_exclude_patterns, ) - - documents.append(doc) - logger.debug(f"Loaded document: {clean_path} ({language})") - - logger.info(f"Loaded {len(documents)} documents from {repo_path} (excluded {excluded_count} files by patterns)") - return documents + ) + return self.load_file_batch( + file_paths, + repo_path, + workspace, + project, + branch, + commit, + ) def load_specific_files( self, @@ -306,31 +297,26 @@ def load_specific_files( full_path = repo_base / relative_file_path relative_path = str(relative_file_path) - if not full_path.exists(): - logger.warning(f"File does not exist: {full_path} (relative: {relative_path})") - continue - - if not full_path.is_file(): - continue - if should_exclude_file(relative_path, self.config.excluded_patterns): logger.debug(f"Excluding file: {relative_path}") continue - if full_path.stat().st_size > self.config.max_file_size_bytes: - logger.warning(f"File too large, skipping: {relative_path}") - continue - - if is_binary_file(full_path): - logger.debug(f"Binary file, skipping: {relative_path}") - continue - if _is_generated_asset(full_path.name): logger.debug(f"Generated asset, skipping: {relative_path}") continue try: - text = full_path.read_text(encoding="utf-8") + content = read_repository_file_bytes( + repo_base, + relative_file_path, + ) + if len(content) > self.config.max_file_size_bytes: + logger.warning(f"File too large, skipping: {relative_path}") + continue + text = _decode_text(content) + if text is None: + logger.debug(f"Binary file, skipping: {relative_path}") + continue except Exception as e: logger.error(f"Error reading file {relative_path}: {e}") continue diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_identity.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_identity.py index a6f360740..6108bb9c0 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_identity.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_identity.py @@ -47,6 +47,7 @@ def pr_overlay_generation_fingerprint( index_representation_fingerprint: str, pr_overlay_representation_fingerprint: str, snapshots: Iterable[object], + base_generation_manifest_sha256: str = "", ) -> str: """Hash every input that can change persisted semantic or plugin context.""" digest = hashlib.sha256() @@ -59,6 +60,10 @@ def pr_overlay_generation_fingerprint( ("base_branch", base_branch), ("source_revision", source_revision), ("base_revision", base_revision), + ( + "base_generation_manifest_sha256", + base_generation_manifest_sha256, + ), ("request_plugin_fingerprint", request_plugin_fingerprint), ("target_plugin_fingerprint", target_plugin_fingerprint), ("capability_fingerprint", capability_fingerprint), diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_manifest.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_manifest.py new file mode 100644 index 000000000..8d34e1b60 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_manifest.py @@ -0,0 +1,374 @@ +"""Content-addressed completeness seals for persisted pull-request overlays.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from typing import Any + +from llama_index.core.schema import TextNode +from qdrant_client.models import FieldCondition, Filter, MatchValue + +from .generation_manifest import ( + GenerationManifestError, + compute_generation_members_digest, + verified_generation_member, +) +from .repository_overlay import IncrementalIndexPreconditionError + + +PR_OVERLAY_MANIFEST_PAYLOAD_KEY = "pr_overlay_generation_manifest" +PR_OVERLAY_SCHEMA = "codecrow.pr-overlay-generation" +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_FINGERPRINT_RE = re.compile(r"^sha256:[0-9a-f]{64}$") + + +def _canonical_json(value: Any) -> str: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def pr_overlay_manifest_path(pr_number: int) -> str: + return ( + "__analysis_state__/pr-overlay-generation/" + f"{pr_number}/000000.state" + ) + + +def pr_overlay_manifest_content( + *, + workspace: str, + project: str, + pr_number: int, + branch: str, + base_branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, + member_count: int, + members_sha256: str, +) -> str: + return _canonical_json({ + "baseBranch": base_branch, + "baseGenerationManifestSha256": base_generation_manifest_sha256, + "baseRevision": base_revision, + "branch": branch, + "generationFingerprint": generation_fingerprint, + "memberCount": member_count, + "membersSha256": members_sha256, + "overlayRepresentationFingerprint": ( + overlay_representation_fingerprint + ), + "prNumber": pr_number, + "project": project, + "schema": PR_OVERLAY_SCHEMA, + "sourceRevision": source_revision, + "workspace": workspace, + }) + + +def build_pr_overlay_manifest_node( + *, + workspace: str, + project: str, + pr_number: int, + branch: str, + base_branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, + members: Sequence[tuple[object, str]], + identity_metadata: Mapping[str, Any], +) -> tuple[TextNode, dict[str, Any]]: + members_sha256 = compute_generation_members_digest(members) + content = pr_overlay_manifest_content( + workspace=workspace, + project=project, + pr_number=pr_number, + branch=branch, + base_branch=base_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_manifest_sha256 + ), + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + member_count=len(members), + members_sha256=members_sha256, + ) + manifest_sha256 = hashlib.sha256(content.encode("utf-8")).hexdigest() + metadata = { + "workspace": workspace, + "project": project, + "branch": branch, + "path": pr_overlay_manifest_path(pr_number), + "language": "repository-state", + "filetype": "state", + "pr": True, + "pr_number": pr_number, + "pr_branch": branch, + "pr_source_revision": source_revision, + "pr_base_revision": base_revision, + "pr_base_generation_manifest_sha256": ( + base_generation_manifest_sha256 + ), + "pr_generation_fingerprint": generation_fingerprint, + PR_OVERLAY_MANIFEST_PAYLOAD_KEY: True, + "pr_overlay_generation_schema": PR_OVERLAY_SCHEMA, + "pr_overlay_generation_member_count": len(members), + "pr_overlay_generation_members_sha256": members_sha256, + "pr_overlay_generation_manifest_sha256": manifest_sha256, + "pr_overlay_base_branch": base_branch, + "pr_overlay_representation_fingerprint": ( + overlay_representation_fingerprint + ), + **dict(identity_metadata), + } + return TextNode(text=content, metadata=metadata), { + "overlay_generation_member_count": len(members), + "overlay_generation_members_sha256": members_sha256, + "overlay_generation_manifest_sha256": manifest_sha256, + } + + +def _pr_filter( + *, + workspace: str, + project: str, + pr_number: int, + branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, +) -> Filter: + return Filter(must=[ + FieldCondition(key="pr", match=MatchValue(value=True)), + FieldCondition( + key="workspace", + match=MatchValue(value=workspace), + ), + FieldCondition( + key="project", + match=MatchValue(value=project), + ), + FieldCondition( + key="pr_number", + match=MatchValue(value=pr_number), + ), + FieldCondition( + key="branch", + match=MatchValue(value=branch), + ), + FieldCondition( + key="pr_source_revision", + match=MatchValue(value=source_revision), + ), + FieldCondition( + key="pr_base_revision", + match=MatchValue(value=base_revision), + ), + FieldCondition( + key="pr_base_generation_manifest_sha256", + match=MatchValue(value=base_generation_manifest_sha256), + ), + FieldCondition( + key="pr_generation_fingerprint", + match=MatchValue(value=generation_fingerprint), + ), + FieldCondition( + key="pr_overlay_representation_fingerprint", + match=MatchValue(value=overlay_representation_fingerprint), + ), + ]) + + +def read_pr_overlay_generation( + client, + collection_name: str, + *, + workspace: str, + project: str, + pr_number: int, + branch: str, + base_branch: str, + source_revision: str, + base_revision: str, + base_generation_manifest_sha256: str, + generation_fingerprint: str, + overlay_representation_fingerprint: str, + expected_manifest_sha256: str | None = None, +) -> dict[str, Any] | None: + """Read and verify every member of one exact PR overlay generation.""" + points = [] + offset = None + while True: + batch, offset = client.scroll( + collection_name=collection_name, + scroll_filter=_pr_filter( + workspace=workspace, + project=project, + pr_number=pr_number, + branch=branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_manifest_sha256 + ), + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + ), + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + points.extend(batch) + if offset is None: + break + if not points: + return None + + manifest_points = [ + point + for point in points + if (point.payload or {}).get(PR_OVERLAY_MANIFEST_PAYLOAD_KEY) is True + ] + if len(manifest_points) != 1: + raise IncrementalIndexPreconditionError( + "PR overlay generation manifest is missing or not unique" + ) + manifest = manifest_points[0].payload or {} + members = [ + point + for point in points + if (point.payload or {}).get(PR_OVERLAY_MANIFEST_PAYLOAD_KEY) is not True + ] + expected_identity = { + "workspace": workspace, + "project": project, + "branch": branch, + "pr_number": pr_number, + "pr_source_revision": source_revision, + "pr_base_revision": base_revision, + "pr_base_generation_manifest_sha256": ( + base_generation_manifest_sha256 + ), + "pr_generation_fingerprint": generation_fingerprint, + "pr_overlay_base_branch": base_branch, + "pr_overlay_representation_fingerprint": ( + overlay_representation_fingerprint + ), + } + if any( + manifest.get(field) != expected + for field, expected in expected_identity.items() + ): + raise IncrementalIndexPreconditionError( + "PR overlay generation manifest identity does not match the request" + ) + manifest_sha256 = manifest.get( + "pr_overlay_generation_manifest_sha256" + ) + member_count = manifest.get("pr_overlay_generation_member_count") + members_sha256 = manifest.get("pr_overlay_generation_members_sha256") + if ( + manifest.get("pr_overlay_generation_schema") != PR_OVERLAY_SCHEMA + or manifest.get("path") != pr_overlay_manifest_path(pr_number) + or type(member_count) is not int + or member_count < 0 + or not isinstance(members_sha256, str) + or _SHA256_RE.fullmatch(members_sha256) is None + or not isinstance(manifest_sha256, str) + or _SHA256_RE.fullmatch(manifest_sha256) is None + or ( + expected_manifest_sha256 is not None + and manifest_sha256 != expected_manifest_sha256 + ) + ): + raise IncrementalIndexPreconditionError( + "PR overlay generation manifest is invalid" + ) + if len(members) != member_count: + raise IncrementalIndexPreconditionError( + "PR overlay generation membership is incomplete" + ) + for point in members: + payload = point.payload or {} + if any( + payload.get(field) != expected + for field, expected in expected_identity.items() + if field not in {"pr_overlay_base_branch"} + ): + raise IncrementalIndexPreconditionError( + "PR overlay contains a member outside the sealed generation" + ) + if payload.get(PR_OVERLAY_MANIFEST_PAYLOAD_KEY) is True: + raise IncrementalIndexPreconditionError( + "PR overlay manifest was included as an ordinary member" + ) + try: + observed_members = [ + verified_generation_member(point) for point in members + ] + observed_members_sha256 = compute_generation_members_digest( + observed_members + ) + except GenerationManifestError as exception: + raise IncrementalIndexPreconditionError( + "PR overlay generation member integrity failed" + ) from exception + if observed_members_sha256 != members_sha256: + raise IncrementalIndexPreconditionError( + "PR overlay generation membership digest does not match its seal" + ) + expected_content = pr_overlay_manifest_content( + workspace=workspace, + project=project, + pr_number=pr_number, + branch=branch, + base_branch=base_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest_sha256=( + base_generation_manifest_sha256 + ), + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=( + overlay_representation_fingerprint + ), + member_count=member_count, + members_sha256=members_sha256, + ) + if hashlib.sha256(expected_content.encode("utf-8")).hexdigest() != ( + manifest_sha256 + ): + raise IncrementalIndexPreconditionError( + "PR overlay generation manifest content failed integrity validation" + ) + return { + "overlay_generation_member_count": member_count, + "overlay_generation_members_sha256": members_sha256, + "overlay_generation_manifest_sha256": manifest_sha256, + } + + +def is_pr_overlay_fingerprint(value: object) -> bool: + return isinstance(value, str) and _FINGERPRINT_RE.fullmatch(value) is not None diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_representation.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_representation.py index 57cf5fcc3..94e13d0f4 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_representation.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/pr_overlay_representation.py @@ -18,10 +18,19 @@ _PR_OVERLAY_SOURCE_PATHS = ( "api/models.py", "api/routers/pr.py", + "api/routers/query.py", "core/index_manager/manager.py", "core/pr_overlay_identity.py", + "core/pr_overlay_manifest.py", "core/pr_overlay_representation.py", + "core/revision_binding.py", + "core/revision_preflight.py", "core/review_grouping.py", + "services/base.py", + "services/deterministic_context.py", + "services/pr_context.py", + "services/query_service.py", + "services/semantic_search.py", ) _PR_OVERLAY_DEPENDENCIES = ( diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_binding.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_binding.py new file mode 100644 index 000000000..d089a479d --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_binding.py @@ -0,0 +1,78 @@ +"""Fail-closed repository-generation leases for review retrieval.""" + +from __future__ import annotations + +from .repository_overlay import IncrementalIndexPreconditionError + + +def require_repository_generation( + index_manager, + *, + workspace: str, + project: str, + branch: str, + revision: str, + generation_manifest_sha256: str | None = None, + collection_target: str | None = None, +): + """Load one exact sealed generation and optionally match its receipt.""" + logical_collection = index_manager._get_project_collection_name( + workspace, + project, + ) + requested_target = collection_target or logical_collection + active_target = index_manager._collection_manager.resolve_collection_target( + requested_target + ) + if active_target is None: + raise IncrementalIndexPreconditionError( + "requested repository collection is unavailable" + ) + bound_target = active_target + result = index_manager.get_revision_preflight( + workspace, + project, + branch, + revision, + collection_target=bound_target, + ) + if result is None: + raise IncrementalIndexPreconditionError( + "requested repository revision is not available as one complete " + f"sealed generation: {branch}@{revision}" + ) + if ( + generation_manifest_sha256 is not None + and result["generation_manifest_sha256"] + != generation_manifest_sha256 + ): + raise IncrementalIndexPreconditionError( + "requested repository generation changed or does not match its " + f"receipt: {branch}@{revision}" + ) + return { + **result, + "_collection_target": bound_target, + "_lease_target": requested_target, + } + + +def require_same_repository_generation( + index_manager, + *, + workspace: str, + project: str, + branch: str, + revision: str, + receipt, +): + """Recheck a generation lease after a non-transactional retrieval/build.""" + return require_repository_generation( + index_manager, + workspace=workspace, + project=project, + branch=branch, + revision=revision, + generation_manifest_sha256=receipt["generation_manifest_sha256"], + collection_target=receipt["_lease_target"], + ) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py new file mode 100644 index 000000000..bda343f51 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py @@ -0,0 +1,478 @@ +"""Read-only verification for immutable repository index revisions.""" + +from __future__ import annotations + +import hashlib +import json + +from qdrant_client.models import FieldCondition, Filter, MatchValue + +from .generation_manifest import ( + GENERATION_MANIFEST_PATH, + GENERATION_MANIFEST_PAYLOAD_KEY, + GENERATION_SCHEMA, + GenerationManifestError, + canonical_index_selection_policy, + compute_index_selection_policy_sha256, + compute_generation_members_digest, + generation_manifest_content, + is_sha256_hex, + verified_generation_member, +) +from .index_representation import INDEX_REPRESENTATION_PAYLOAD_KEY +from .repository_overlay import ( + IncrementalIndexPreconditionError, + scroll_branch_points, +) + + +_IDENTITY_PAYLOAD_FIELDS = ( + "plugin_ids", + "plugin_fingerprint", + "plugin_descriptor_fingerprint", + "plugin_implementation_fingerprint", + INDEX_REPRESENTATION_PAYLOAD_KEY, +) + + +def _payload_identity(payload): + plugin_ids = payload.get("plugin_ids") + fingerprints = tuple( + payload.get(field) for field in _IDENTITY_PAYLOAD_FIELDS[1:] + ) + if ( + not isinstance(plugin_ids, (list, tuple)) + or not all( + isinstance(plugin_id, str) and plugin_id + for plugin_id in plugin_ids + ) + or not all( + isinstance(fingerprint, str) and fingerprint + for fingerprint in fingerprints + ) + ): + raise IncrementalIndexPreconditionError( + "exact revision point is missing repository build identity; " + "fully reindex the revision" + ) + return (tuple(plugin_ids), *fingerprints) + + +def _load_exact_repository_facts( + client, + collection_name: str, + branch: str, + commit: str, +): + """Load and integrity-check the facts sentinel for one exact revision.""" + from codecrow_plugins import RepositoryFacts + + points = scroll_branch_points( + client, + collection_name, + branch, + ( + FieldCondition(key="commit", match=MatchValue(value=commit)), + FieldCondition( + key="repository_facts_state", + match=MatchValue(value=True), + ), + ), + ) + if not points: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts are missing; " + "fully reindex the revision" + ) + + payloads = [point.payload or {} for point in points] + for payload in payloads: + if ( + type(payload.get("facts_part")) is not int + or payload["facts_part"] < 0 + or type(payload.get("facts_parts")) is not int + or payload["facts_parts"] < 1 + ): + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts have invalid part " + "metadata; fully reindex the revision" + ) + ordered = sorted(payloads, key=lambda payload: payload.get("facts_part", -1)) + expected_parts = ordered[0].get("facts_parts") + actual_parts = [payload.get("facts_part") for payload in ordered] + if ( + actual_parts != list(range(expected_parts)) + or any(payload.get("facts_parts") != expected_parts for payload in ordered) + ): + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts are incomplete; " + "fully reindex the revision" + ) + + expected_digest = ordered[0].get("facts_content_sha256") + if ( + not isinstance(expected_digest, str) + or len(expected_digest) != 64 + or any( + character not in "0123456789abcdef" + for character in expected_digest + ) + or any( + payload.get("facts_content_sha256") != expected_digest + for payload in ordered + ) + ): + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts have invalid integrity " + "metadata; fully reindex the revision" + ) + + content_parts = [ + payload.get("text", payload.get("_node_content", "")) + for payload in ordered + ] + if not all(isinstance(part, str) for part in content_parts): + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts have invalid content; " + "fully reindex the revision" + ) + content = "".join(content_parts) + if hashlib.sha256(content.encode("utf-8")).hexdigest() != expected_digest: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts failed integrity " + "validation; fully reindex the revision" + ) + + state_identity = None + for payload in ordered: + candidate = _payload_identity(payload) + if state_identity is None: + state_identity = candidate + elif state_identity != candidate: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts have inconsistent " + "build identity; fully reindex the revision" + ) + + try: + decoded = json.loads(content) + repository_facts = RepositoryFacts( + revision=decoded["revision"], + paths=tuple(decoded["paths"]), + marker_contents=decoded.get("markerContents", {}), + ) + except Exception as exception: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts are invalid; " + "fully reindex the revision" + ) from exception + if repository_facts.revision != commit: + raise IncrementalIndexPreconditionError( + "exact revision repository detection facts do not match the " + "requested commit; fully reindex the revision" + ) + return repository_facts, expected_digest, state_identity + + +def _validate_generation_manifest( + manifest_points, + members, + branch: str, + commit: str, +): + """Validate the single seal against every observed non-manifest point.""" + if len(manifest_points) != 1: + reason = "missing" if not manifest_points else "not unique" + raise IncrementalIndexPreconditionError( + f"exact revision repository generation manifest is {reason}; " + "fully reindex the revision" + ) + + manifest_payload = manifest_points[0].payload or {} + expected_count = manifest_payload.get("generation_member_count") + expected_members_digest = manifest_payload.get( + "generation_members_sha256" + ) + expected_manifest_digest = manifest_payload.get( + "generation_manifest_sha256" + ) + source_tree_sha256 = manifest_payload.get("source_tree_sha256") + index_include_patterns = manifest_payload.get("index_include_patterns") + index_exclude_patterns = manifest_payload.get("index_exclude_patterns") + index_selection_policy_sha256 = manifest_payload.get( + "index_selection_policy_sha256" + ) + workspace = manifest_payload.get("workspace") + project = manifest_payload.get("project") + if ( + manifest_payload.get("generation_schema") != GENERATION_SCHEMA + or manifest_payload.get("path") != GENERATION_MANIFEST_PATH + or not isinstance(workspace, str) + or not workspace + or not isinstance(project, str) + or not project + or type(expected_count) is not int + or expected_count < 1 + or not is_sha256_hex(expected_members_digest) + or not is_sha256_hex(expected_manifest_digest) + or not is_sha256_hex(source_tree_sha256) + or not isinstance(index_include_patterns, list) + or not isinstance(index_exclude_patterns, list) + or not is_sha256_hex(index_selection_policy_sha256) + ): + raise IncrementalIndexPreconditionError( + "exact revision repository generation manifest is invalid; " + "fully reindex the revision" + ) + try: + selection_policy = canonical_index_selection_policy( + index_include_patterns, + index_exclude_patterns, + ) + observed_selection_policy_sha256 = ( + compute_index_selection_policy_sha256( + selection_policy["includePatterns"], + selection_policy["excludePatterns"], + ) + ) + except GenerationManifestError as exception: + raise IncrementalIndexPreconditionError( + "exact revision repository index selection policy is invalid; " + "fully reindex the revision" + ) from exception + if ( + index_include_patterns != selection_policy["includePatterns"] + or index_exclude_patterns != selection_policy["excludePatterns"] + or index_selection_policy_sha256 + != observed_selection_policy_sha256 + ): + raise IncrementalIndexPreconditionError( + "exact revision repository index selection policy failed " + "integrity validation; fully reindex the revision" + ) + if expected_count != len(members): + raise IncrementalIndexPreconditionError( + "exact revision repository generation is incomplete: " + f"expected={expected_count}, actual={len(members)}; " + "fully reindex the revision" + ) + + try: + observed_members_digest = compute_generation_members_digest(members) + except GenerationManifestError as exception: + raise IncrementalIndexPreconditionError( + "exact revision repository generation members are invalid; " + "fully reindex the revision" + ) from exception + if observed_members_digest != expected_members_digest: + raise IncrementalIndexPreconditionError( + "exact revision repository generation membership failed integrity " + "validation; fully reindex the revision" + ) + + expected_content = generation_manifest_content( + workspace=workspace, + project=project, + branch=branch, + commit=commit, + member_count=expected_count, + members_sha256=expected_members_digest, + source_tree_sha256=source_tree_sha256, + index_include_patterns=index_include_patterns, + index_exclude_patterns=index_exclude_patterns, + index_selection_policy_sha256=index_selection_policy_sha256, + ) + if ( + hashlib.sha256(expected_content.encode("utf-8")).hexdigest() + != expected_manifest_digest + ): + raise IncrementalIndexPreconditionError( + "exact revision repository generation manifest failed integrity " + "validation; fully reindex the revision" + ) + return { + "workspace": workspace, + "project": project, + "generation_schema": GENERATION_SCHEMA, + "generation_member_count": expected_count, + "generation_members_sha256": expected_members_digest, + "generation_manifest_sha256": expected_manifest_digest, + "source_tree_sha256": source_tree_sha256, + "index_include_patterns": index_include_patterns, + "index_exclude_patterns": index_exclude_patterns, + "index_selection_policy_sha256": index_selection_policy_sha256, + } + + +def _require_unmixed_branch_revision( + client, + collection_name: str, + branch: str, + commit: str, +) -> None: + """Reject a branch whose non-PR retrieval population spans revisions.""" + branch_filter = Filter( + must=[ + FieldCondition(key="branch", match=MatchValue(value=branch)), + ], + must_not=[ + FieldCondition(key="pr", match=MatchValue(value=True)), + ], + ) + offset = None + while True: + points, offset = client.scroll( + collection_name=collection_name, + scroll_filter=branch_filter, + limit=256, + offset=offset, + with_payload=["branch", "commit", "pr"], + with_vectors=False, + ) + for point in points: + payload = point.payload or {} + if ( + payload.get("branch") != branch + or payload.get("pr") is True + or payload.get("commit") != commit + ): + raise IncrementalIndexPreconditionError( + "exact revision branch contains mixed repository revisions; " + "fully reindex the revision" + ) + if offset is None: + break + + +def read_repository_revision_preflight( + client, + collection_name: str, + branch: str, + commit: str, +): + """Return a verified exact repository revision or ``None`` when absent. + + A revision is reusable only when every matching repository point carries + one consistent representation/plugin identity, its content-addressed + generation membership matches the atomic full-index seal, and its + repository-facts sentinel is complete, digest-valid, and bound to the + requested commit. Qdrant failures intentionally propagate so callers + cannot mistake an unavailable store for a missing revision. + """ + revision_filter = Filter( + must=[ + FieldCondition(key="branch", match=MatchValue(value=branch)), + FieldCondition(key="commit", match=MatchValue(value=commit)), + ], + must_not=[ + FieldCondition(key="pr", match=MatchValue(value=True)), + ], + ) + point_count = 0 + identity = None + manifest_points = [] + member_points = [] + offset = None + + while True: + points, offset = client.scroll( + collection_name=collection_name, + scroll_filter=revision_filter, + limit=256, + offset=offset, + with_payload=True, + with_vectors=True, + ) + point_count += len(points) + for point in points: + payload = point.payload or {} + if ( + payload.get("branch") != branch + or payload.get("commit") != commit + or payload.get("pr") is True + ): + raise IncrementalIndexPreconditionError( + "exact revision query returned a point outside the requested " + "repository snapshot" + ) + candidate = _payload_identity(payload) + if identity is None: + identity = candidate + elif identity != candidate: + raise IncrementalIndexPreconditionError( + "exact revision has inconsistent repository build identity; " + "fully reindex the revision" + ) + if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: + manifest_points.append(point) + else: + member_points.append(point) + if offset is None: + break + + if point_count == 0: + return None + + _require_unmixed_branch_revision( + client, + collection_name, + branch, + commit, + ) + if len(manifest_points) != 1: + _validate_generation_manifest( + manifest_points, + [], + branch, + commit, + ) + try: + members = [ + verified_generation_member(point) + for point in member_points + ] + except GenerationManifestError as exception: + raise IncrementalIndexPreconditionError( + "exact revision repository generation member content failed " + "integrity validation; fully reindex the revision" + ) from exception + generation_identity = _validate_generation_manifest( + manifest_points, + members, + branch, + commit, + ) + repository_facts, facts_digest, state_identity = ( + _load_exact_repository_facts( + client, + collection_name, + branch, + commit, + ) + ) + if identity != state_identity: + raise IncrementalIndexPreconditionError( + "exact revision points do not match repository detection build " + "identity; fully reindex the revision" + ) + + ( + plugin_ids, + plugin_fingerprint, + descriptor_fingerprint, + implementation_fingerprint, + representation_fingerprint, + ) = identity + return { + "branch": branch, + "commit": commit, + "point_count": point_count, + "repository_revision": repository_facts.revision, + "repository_facts_sha256": facts_digest, + "plugin_ids": list(plugin_ids), + "plugin_fingerprint": plugin_fingerprint, + "plugin_descriptor_fingerprint": descriptor_fingerprint, + "plugin_implementation_fingerprint": implementation_fingerprint, + "index_representation_fingerprint": representation_fingerprint, + **generation_identity, + } diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/source_tree.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/source_tree.py new file mode 100644 index 000000000..684c1c563 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/source_tree.py @@ -0,0 +1,382 @@ +"""Immutable source-tree verification for repository indexing.""" + +from __future__ import annotations + +import hashlib +import os +import re +import stat +import subprocess +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import BinaryIO, Mapping + + +SOURCE_TREE_SCHEMA = "codecrow.repository-source-tree" +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +class RepositorySourceTreeError(RuntimeError): + """The indexing source is not the attested immutable repository tree.""" + + +@dataclass(frozen=True) +class RepositorySourceTree: + """Verified source identity retained across the indexing operation.""" + + commit: str + tree_sha256: str + git_commit_verified: bool + file_sha256_by_path: Mapping[str, str] + + +def _feed_framed(hasher, value: bytes) -> None: + hasher.update(len(value).to_bytes(8, "big")) + hasher.update(value) + + +def _relative_parts(relative_path: str | Path) -> tuple[str, ...]: + path = Path(relative_path) + parts = path.parts + if ( + path.is_absolute() + or not parts + or any(part in {"", ".", ".."} for part in parts) + ): + raise RepositorySourceTreeError( + f"invalid repository-relative source path: {relative_path}" + ) + return tuple(parts) + + +@contextmanager +def open_repository_file_no_follow( + repo_path: str | Path, + relative_path: str | Path, +) -> BinaryIO: + """Open one regular repository file through pinned, no-follow descriptors.""" + parts = _relative_parts(relative_path) + directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + file_flags = os.O_RDONLY + no_follow = getattr(os, "O_NOFOLLOW", 0) + if not no_follow: + raise RepositorySourceTreeError( + "repository source verification requires O_NOFOLLOW support" + ) + directory_flags |= no_follow + file_flags |= no_follow + close_on_exec = getattr(os, "O_CLOEXEC", 0) + directory_flags |= close_on_exec + file_flags |= close_on_exec + + directory_fd = None + file_fd = None + try: + directory_fd = os.open(os.fspath(repo_path), directory_flags) + for component in parts[:-1]: + child_fd = os.open( + component, + directory_flags, + dir_fd=directory_fd, + ) + os.close(directory_fd) + directory_fd = child_fd + file_fd = os.open(parts[-1], file_flags, dir_fd=directory_fd) + file_stat = os.fstat(file_fd) + if not stat.S_ISREG(file_stat.st_mode): + raise RepositorySourceTreeError( + "repository source entry is not a regular file: " + + Path(*parts).as_posix() + ) + with os.fdopen(file_fd, "rb", closefd=True) as source: + file_fd = None + yield source + except OSError as exception: + raise RepositorySourceTreeError( + "cannot safely open repository source file: " + + Path(*parts).as_posix() + ) from exception + finally: + if file_fd is not None: + os.close(file_fd) + if directory_fd is not None: + os.close(directory_fd) + + +def read_repository_file_bytes( + repo_path: str | Path, + relative_path: str | Path, + *, + expected_sha256: str | None = None, +) -> bytes: + """Read one regular file without symlink traversal and verify its identity.""" + with open_repository_file_no_follow(repo_path, relative_path) as source: + content = source.read() + if ( + expected_sha256 is not None + and hashlib.sha256(content).hexdigest() != expected_sha256 + ): + raise RepositorySourceTreeError( + "repository source file changed after attestation: " + + Path(relative_path).as_posix() + ) + return content + + +def _repository_entries(root: Path): + """Yield non-Git source entries without following repository symlinks.""" + + collected_entries = [] + directory_flags = ( + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + if not getattr(os, "O_NOFOLLOW", 0): + raise RepositorySourceTreeError( + "repository source verification requires O_NOFOLLOW support" + ) + + def visit(directory_fd: int, relative_directory: Path): + try: + with os.scandir(directory_fd) as scanner: + entries = list(scanner) + except OSError as exception: + raise RepositorySourceTreeError( + f"cannot enumerate repository source directory: {relative_directory}" + ) from exception + + for entry in entries: + relative_path = relative_directory / entry.name + if relative_directory == Path() and entry.name == ".git": + continue + try: + entry_stat = os.stat( + entry.name, + dir_fd=directory_fd, + follow_symlinks=False, + ) + if stat.S_ISLNK(entry_stat.st_mode): + collected_entries.append( + ( + "symlink", + relative_path, + os.readlink(entry.name, dir_fd=directory_fd), + ) + ) + elif stat.S_ISDIR(entry_stat.st_mode): + child_fd = os.open( + entry.name, + directory_flags, + dir_fd=directory_fd, + ) + try: + visit(child_fd, relative_path) + finally: + os.close(child_fd) + elif stat.S_ISREG(entry_stat.st_mode): + collected_entries.append( + ("file", relative_path, None) + ) + else: + raise RepositorySourceTreeError( + "repository source contains an unsupported filesystem " + f"entry: {relative_path.as_posix()}" + ) + except OSError as exception: + raise RepositorySourceTreeError( + "cannot inspect repository source entry: " + f"{relative_path.as_posix()}" + ) from exception + + root_fd = None + try: + root_fd = os.open(os.fspath(root), directory_flags) + visit(root_fd, Path()) + except OSError as exception: + raise RepositorySourceTreeError( + f"cannot safely enumerate repository source directory: {root}" + ) from exception + finally: + if root_fd is not None: + os.close(root_fd) + collected_entries.sort( + key=lambda item: item[1] + .as_posix() + .encode("utf-8", "surrogateescape") + ) + yield from collected_entries + + +def iter_repository_regular_file_paths( + repo_path: str | Path, +): + """Yield regular-file paths from a no-follow repository traversal.""" + for kind, relative_path, _ in _repository_entries(Path(repo_path)): + if kind == "file": + yield relative_path + + +def _compute_repository_source_tree( + repo_path: str | Path, +) -> tuple[str, Mapping[str, str]]: + """Hash exact repository-relative paths and retain regular-file identities.""" + root = Path(repo_path) + try: + root_stat = root.lstat() + except OSError as exception: + raise RepositorySourceTreeError( + f"repository source path is not a directory: {root}" + ) from exception + if not stat.S_ISDIR(root_stat.st_mode) or stat.S_ISLNK(root_stat.st_mode): + raise RepositorySourceTreeError( + f"repository source path is not a directory: {root}" + ) + + hasher = hashlib.sha256() + _feed_framed(hasher, SOURCE_TREE_SCHEMA.encode("ascii")) + entry_count = 0 + file_sha256_by_path: dict[str, str] = {} + for kind, relative_path, value in _repository_entries(root): + entry_count += 1 + _feed_framed(hasher, kind.encode("ascii")) + _feed_framed( + hasher, + relative_path.as_posix().encode("utf-8", "surrogateescape"), + ) + if kind == "symlink": + _feed_framed( + hasher, + value.encode("utf-8", "surrogateescape"), + ) + continue + + try: + observed_size = 0 + file_hasher = hashlib.sha256() + with open_repository_file_no_follow(root, relative_path) as source: + expected_size = os.fstat(source.fileno()).st_size + hasher.update(expected_size.to_bytes(8, "big")) + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + observed_size += len(chunk) + hasher.update(chunk) + file_hasher.update(chunk) + except (OSError, RepositorySourceTreeError) as exception: + raise RepositorySourceTreeError( + "cannot read repository source file: " + f"{relative_path.as_posix()}" + ) from exception + if observed_size != expected_size: + raise RepositorySourceTreeError( + "repository source changed while it was being attested: " + f"{relative_path.as_posix()}" + ) + file_sha256_by_path[relative_path.as_posix()] = file_hasher.hexdigest() + + hasher.update(entry_count.to_bytes(8, "big")) + return hasher.hexdigest(), MappingProxyType(file_sha256_by_path) + + +def compute_repository_source_tree_sha256(repo_path: str | Path) -> str: + """Hash exact repository-relative paths and bytes deterministically.""" + return _compute_repository_source_tree(repo_path)[0] + + +def _git_output(root: Path, *arguments: str) -> str: + environment = dict(os.environ) + environment["GIT_OPTIONAL_LOCKS"] = "0" + try: + result = subprocess.run( + ["git", "-C", str(root), *arguments], + check=True, + capture_output=True, + text=True, + timeout=30, + env=environment, + ) + except (OSError, subprocess.SubprocessError) as exception: + raise RepositorySourceTreeError( + "cannot verify repository Git revision" + ) from exception + return result.stdout + + +def _verify_git_checkout(root: Path, commit: str) -> bool: + """Require exact HEAD and no tracked or untracked working-tree changes.""" + git_marker = root / ".git" + if not git_marker.exists(): + return False + + observed_commit = _git_output(root, "rev-parse", "--verify", "HEAD^{commit}").strip() + if observed_commit != commit: + raise RepositorySourceTreeError( + "repository Git HEAD does not match the supplied commit: " + f"expected={commit}, actual={observed_commit}" + ) + status = _git_output( + root, + "status", + "--porcelain=v1", + "--untracked-files=all", + ) + if status: + raise RepositorySourceTreeError( + "repository Git working tree is not clean for the supplied commit" + ) + return True + + +def verify_repository_source_tree( + repo_path: str | Path, + commit: str, + expected_tree_sha256: str, +) -> RepositorySourceTree: + """Verify the caller-attested tree and, for Git worktrees, exact HEAD.""" + if not isinstance(commit, str) or not commit: + raise RepositorySourceTreeError("repository source commit is required") + if ( + not isinstance(expected_tree_sha256, str) + or not _SHA256_RE.fullmatch(expected_tree_sha256) + ): + raise RepositorySourceTreeError( + "repository source tree requires a canonical SHA-256 attestation" + ) + + root = Path(repo_path) + git_commit_verified = _verify_git_checkout(root, commit) + observed_tree_sha256, file_sha256_by_path = _compute_repository_source_tree( + root + ) + if observed_tree_sha256 != expected_tree_sha256: + raise RepositorySourceTreeError( + "repository source tree does not match its acquisition attestation: " + f"expected={expected_tree_sha256}, actual={observed_tree_sha256}" + ) + return RepositorySourceTree( + commit=commit, + tree_sha256=observed_tree_sha256, + git_commit_verified=git_commit_verified, + file_sha256_by_path=file_sha256_by_path, + ) + + +def require_repository_source_tree_unchanged( + repo_path: str | Path, + source_tree: RepositorySourceTree, +) -> None: + """Recheck the exact source just before its generation is sealed.""" + verified = verify_repository_source_tree( + repo_path, + source_tree.commit, + source_tree.tree_sha256, + ) + if verified.git_commit_verified != source_tree.git_commit_verified: + raise RepositorySourceTreeError( + "repository source verification mode changed during indexing" + ) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py index ca46d9c45..7a81a2519 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py @@ -254,3 +254,6 @@ class IndexStats(BaseModel): workspace: str project: str branch: str + generation_manifest_sha256: Optional[str] = None + source_tree_sha256: Optional[str] = None + collection_target: Optional[str] = None diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py index cee27af4e..9926393ac 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/server/rag_queue_consumer.py @@ -158,6 +158,7 @@ async def _handle_job(self, payload_str: str): # index_manager.index_repository is synchronous, so we run it in an executor loop = asyncio.get_running_loop() progress_delivery_available = True + progress_futures = [] def publish_progress(event: Dict[str, Any]) -> None: nonlocal progress_delivery_available @@ -168,17 +169,10 @@ def publish_progress(event: Dict[str, Any]) -> None: self._publish_event(event_queue_key, payload), loop, ) - # Surface Redis publication failures promptly without coupling - # indexing correctness to progress delivery. - try: - future.result(timeout=5) - except Exception as exception: - progress_delivery_available = False - logger.warning( - "Could not publish RAG progress for job %s: %s", - job_id, - exception, - ) + # Never block the indexing executor waiting for its own event + # loop. Drain these publications before the terminal event so + # ordering is retained and delivery remains fail-open. + progress_futures.append(future) indexing_future = loop.run_in_executor( None, @@ -188,9 +182,11 @@ def publish_progress(event: Dict[str, Any]) -> None: project=request_dto.project, branch=request_dto.branch, commit=request_dto.commit, + source_tree_sha256=request_dto.source_tree_sha256, preserve_other_branches=request_dto.preserve_other_branches, include_patterns=request_dto.include_patterns, exclude_patterns=request_dto.exclude_patterns, + collection_target=request_dto.collection_target, progress_callback=publish_progress, ) ) @@ -209,6 +205,16 @@ def publish_progress(event: Dict[str, Any]) -> None: }) result_obj = await indexing_future + for progress_future in progress_futures: + try: + await asyncio.wrap_future(progress_future) + except Exception as exception: + progress_delivery_available = False + logger.warning( + "Could not publish RAG progress for job %s: %s", + job_id, + exception, + ) # Serialize the IndexStats result to a dictionary result = result_obj.dict() if hasattr(result_obj, "dict") else result_obj.model_dump() diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py index 5635e918a..a2c8b192d 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/base.py @@ -96,7 +96,7 @@ def _collection_or_alias_exists(self, name: str) -> bool: return False def _get_project_collection_name(self, workspace: str, project: str) -> str: - """Generate collection name for a project (single collection for all branches).""" + """Generate the legacy shared collection name for a project.""" namespace = make_project_namespace(workspace, project) return f"{self.config.qdrant_collection_prefix}_{namespace}" diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py index ca0bb9a1a..28ecaa0c2 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py @@ -12,6 +12,8 @@ from qdrant_client.http.models import Filter, FieldCondition, MatchValue, MatchAny from .base import RAGQueryBase +from ..core.pr_overlay_manifest import PR_OVERLAY_MANIFEST_PAYLOAD_KEY +from ..core.repository_overlay import IncrementalIndexPreconditionError from rag_pipeline.utils.path_identity import ( normalize_repository_path, repository_path_suffix_candidates, @@ -235,7 +237,13 @@ def get_deterministic_context( limit_per_file: int = 10, pr_number: Optional[int] = None, pr_changed_files: Optional[List[str]] = None, - additional_identifiers: Optional[List[str]] = None + additional_identifiers: Optional[List[str]] = None, + expected_revisions: Optional[Dict[str, str]] = None, + pr_source_revision: Optional[str] = None, + pr_base_revision: Optional[str] = None, + pr_base_generation_manifest_sha256: Optional[str] = None, + pr_generation_fingerprint: Optional[str] = None, + collection_target: Optional[str] = None, ) -> Dict: """ Get context using DETERMINISTIC metadata-based retrieval. @@ -271,7 +279,10 @@ def get_deterministic_context( Returns: Dict with chunks grouped by retrieval type and rich metadata """ - collection_name = self._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or self._get_project_collection_name(workspace, project) + ) if not self._collection_or_alias_exists(collection_name): logger.warning(f"Collection {collection_name} does not exist") @@ -288,6 +299,11 @@ def get_deterministic_context( }} file_paths = sorted({path.lstrip("/") for path in file_paths if path}) + pr_changed_path_set = { + normalize_repository_path(path) + for path in (pr_changed_files or []) + if normalize_repository_path(path) + } branches = list(dict.fromkeys(branch for branch in branches if branch)) self._observe_branches(collection_name, branches) logger.info(f"Deterministic context: files={file_paths[:5]}, branches={branches}") @@ -295,19 +311,80 @@ def get_deterministic_context( # ── Build branch filter ── target_branch = branches[0] if branches else None + repository_branch_filters = [] + for branch in branches: + conditions = [ + FieldCondition( + key="branch", + match=MatchValue(value=branch), + ), + ] + if expected_revisions and branch in expected_revisions: + conditions.append(FieldCondition( + key="commit", + match=MatchValue(value=expected_revisions[branch]), + )) + repository_branch_filters.append(Filter(must=conditions)) base_branch_condition = ( - FieldCondition(key="branch", match=MatchValue(value=branches[0])) - if len(branches) == 1 - else FieldCondition(key="branch", match=MatchAny(any=branches)) + repository_branch_filters[0] + if len(repository_branch_filters) == 1 + else Filter(should=repository_branch_filters) ) if pr_number: - branch_filter = Filter(should=[ - Filter(must=[base_branch_condition]), - Filter(must=[ - FieldCondition(key="pr", match=MatchValue(value=True)), - FieldCondition(key="pr_number", match=MatchValue(value=pr_number)) + pr_conditions = [ + FieldCondition(key="pr", match=MatchValue(value=True)), + FieldCondition( + key="pr_number", + match=MatchValue(value=pr_number), + ), + ] + if pr_generation_fingerprint: + pr_conditions.extend([ + FieldCondition( + key="pr_source_revision", + match=MatchValue(value=pr_source_revision), + ), + FieldCondition( + key="pr_base_revision", + match=MatchValue(value=pr_base_revision), + ), + FieldCondition( + key="pr_base_generation_manifest_sha256", + match=MatchValue( + value=pr_base_generation_manifest_sha256 + ), + ), + FieldCondition( + key="pr_generation_fingerprint", + match=MatchValue(value=pr_generation_fingerprint), + ), ]) + base_conditions = [base_branch_condition] + base_exclusions = ( + [ + FieldCondition( + key="path", + match=MatchAny(any=sorted(pr_changed_path_set)), + ), + ] + if pr_changed_path_set + else [] + ) + branch_filter = Filter(should=[ + Filter( + must=base_conditions, + must_not=base_exclusions, + ), + Filter( + must=pr_conditions, + must_not=[ + FieldCondition( + key=PR_OVERLAY_MANIFEST_PAYLOAD_KEY, + match=MatchValue(value=True), + ), + ], + ), ]) logger.info(f"Deterministic hybrid mode: also searching PR-indexed data (pr_number={pr_number})") else: @@ -333,9 +410,7 @@ def get_deterministic_context( # The request is the authority for invalidating materialized branch # context. Do not rely on finding a PR-indexed chunk: deleted files and # architecture-only files legitimately have no replacement code chunk. - changed_file_paths = { - path.lstrip("/") for path in (pr_changed_files or []) if path - } + changed_file_paths = set(pr_changed_path_set) seen_texts = set() target_branch_paths = set() @@ -476,6 +551,38 @@ def get_deterministic_context( f"class_ctx: {sum(len(v) for v in class_context.values())}, " f"ns_ctx: {sum(len(v) for v in namespace_context.values())})") + for chunk in all_chunks: + metadata = chunk.get("metadata") or {} + if metadata.get("pr") is True: + if pr_generation_fingerprint and ( + metadata.get("pr_generation_fingerprint") + != pr_generation_fingerprint + or metadata.get("pr_source_revision") + != pr_source_revision + or metadata.get("pr_base_revision") != pr_base_revision + or metadata.get( + "pr_base_generation_manifest_sha256" + ) != pr_base_generation_manifest_sha256 + ): + raise IncrementalIndexPreconditionError( + "deterministic retrieval returned a PR point outside " + "the requested overlay generation" + ) + continue + expected_revision = ( + expected_revisions.get(metadata.get("branch")) + if expected_revisions + else None + ) + if ( + expected_revision is not None + and metadata.get("commit") != expected_revision + ): + raise IncrementalIndexPreconditionError( + "deterministic retrieval returned a repository point " + "outside the requested immutable revision" + ) + return { "chunks": all_chunks, "changed_files": changed_files_chunks, @@ -839,6 +946,20 @@ def _query_changed_file( key=_point_sort_key, ) + # A revision-bound PR request's changed-path manifest is authoritative. + # Modified paths may have an exact overlay member; deleted and + # architecture-only paths legitimately may not. In either case, never + # fall back to the pre-PR target-branch source for that path. + if any( + repository_paths_match(normalized_path, changed_path) + for changed_path in changed_file_paths + ): + results = [ + point + for point in results + if (point.payload or {}).get("pr") is True + ] + # Apply branch priority if target_branch and len(branches) > 1: has_target = any(p.payload.get("branch") == target_branch for p in results) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/pr_context.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/pr_context.py index fca4935b6..c8555bdcb 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/pr_context.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/pr_context.py @@ -12,6 +12,7 @@ from .base import RAGQueryBase from .duplication import generate_duplication_queries +from ..core.repository_overlay import IncrementalIndexPreconditionError from ..models.instructions import InstructionType from ..models.scoring_config import get_scoring_config from ..utils.utils import detect_language_from_path @@ -93,7 +94,9 @@ def get_context_for_pr( min_relevance_score: float = 0.7, base_branch: Optional[str] = None, deleted_files: Optional[List[str]] = None, - exclude_pr_files: Optional[List[str]] = None + exclude_pr_files: Optional[List[str]] = None, + expected_revisions: Optional[Dict[str, str]] = None, + collection_target: Optional[str] = None, ) -> Dict: """ Get relevant context for review using Smart RAG. @@ -117,10 +120,17 @@ def get_context_for_pr( # Determine branches to search branches_to_search = [branch] - collection_name = self._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or self._get_project_collection_name(workspace, project) + ) if not self._collection_or_alias_exists(collection_name): logger.warning(f"Collection {collection_name} does not exist") + if expected_revisions is not None: + raise IncrementalIndexPreconditionError( + "revision-bound PR-context collection is unavailable" + ) return { "relevant_code": [], "related_files": [], @@ -167,7 +177,9 @@ def get_context_for_pr( branches=branches_to_search, top_k=q_top_k, instruction_type=q_instruction_type, - excluded_paths=all_excluded_paths + excluded_paths=all_excluded_paths, + expected_revisions=expected_revisions, + collection_target=collection_name, ) logger.info(f"Query {i+1}/{len(queries)} returned {len(results)} results") diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/semantic_search.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/semantic_search.py index ce7daaf11..bbf28853e 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/semantic_search.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/semantic_search.py @@ -12,6 +12,7 @@ from qdrant_client.http.models import FieldCondition, MatchValue, MatchAny from .base import RAGQueryBase +from ..core.repository_overlay import IncrementalIndexPreconditionError from ..models.instructions import InstructionType, format_query logger = logging.getLogger(__name__) @@ -36,7 +37,9 @@ def semantic_search( branch: str, top_k: int = 10, filter_language: Optional[str] = None, - instruction_type: InstructionType = InstructionType.GENERAL + instruction_type: InstructionType = InstructionType.GENERAL, + expected_revision: Optional[str] = None, + collection_target: Optional[str] = None, ) -> List[Dict]: """Perform semantic search in the repository for a single branch.""" return self.semantic_search_multi_branch( @@ -46,7 +49,11 @@ def semantic_search( branches=[branch], top_k=top_k, filter_language=filter_language, - instruction_type=instruction_type + instruction_type=instruction_type, + expected_revisions=( + {branch: expected_revision} if expected_revision else None + ), + collection_target=collection_target, ) def semantic_search_multi_branch( @@ -58,7 +65,9 @@ def semantic_search_multi_branch( top_k: int = 10, filter_language: Optional[str] = None, instruction_type: InstructionType = InstructionType.GENERAL, - excluded_paths: Optional[List[str]] = None + excluded_paths: Optional[List[str]] = None, + expected_revisions: Optional[Dict[str, str]] = None, + collection_target: Optional[str] = None, ) -> List[Dict]: """Perform semantic search across multiple branches with filtering. @@ -66,7 +75,10 @@ def semantic_search_multi_branch( branches: List of branches to search (e.g., ['feature/xyz', 'main']) excluded_paths: Files to exclude from results (e.g., deleted files) """ - collection_name = self._get_project_collection_name(workspace, project) + collection_name = ( + collection_target + or self._get_project_collection_name(workspace, project) + ) excluded_paths = excluded_paths or [] logger.info(f"Multi-branch search in {collection_name} branches={branches} for: {query[:50]}...") @@ -74,21 +86,42 @@ def semantic_search_multi_branch( try: if not self._collection_or_alias_exists(collection_name): logger.warning(f"Collection {collection_name} does not exist") + if expected_revisions is not None: + raise IncrementalIndexPreconditionError( + "revision-bound semantic-search collection is unavailable" + ) return [] - self._observe_branches(collection_name, branches) + observer = getattr(self, "_observe_branches", None) + if callable(observer): + observer(collection_name, branches) + else: + self._require_compatible_branches(collection_name, branches) # Get or create cached VectorStoreIndex index = self._get_or_create_index(collection_name) # Create retriever with branch filter + if expected_revisions and len(branches) != 1: + raise ValueError( + "revision-bound semantic search requires one authoritative branch" + ) filters = [] for branch in branches: filters.append(MetadataFilter(key="branch", value=branch, operator=FilterOperator.EQ)) + if expected_revisions and branch in expected_revisions: + filters.append(MetadataFilter( + key="commit", + value=expected_revisions[branch], + operator=FilterOperator.EQ, + )) branch_filters = MetadataFilters( filters=filters, - condition="or" if len(filters) > 1 else "and" + condition=( + "and" if expected_revisions + else ("or" if len(filters) > 1 else "and") + ) ) # LlamaIndex's strict MetadataFilter value model does not accept # booleans even though Qdrant payloads do. Retrieve a bounded @@ -118,6 +151,7 @@ def semantic_search_multi_branch( "repository_snapshot", "repository_facts_state", "architecture_source", + "repository_generation_manifest", ) ): continue @@ -143,6 +177,8 @@ def semantic_search_multi_branch( except Exception as e: logger.error(f"Error during multi-branch semantic search: {e}") + if expected_revisions is not None: + raise return [] def _dedupe_by_branch_priority( diff --git a/python-ecosystem/rag-pipeline/tests/test_coordination.py b/python-ecosystem/rag-pipeline/tests/test_coordination.py index c8c542a44..0e52bfeb3 100644 --- a/python-ecosystem/rag-pipeline/tests/test_coordination.py +++ b/python-ecosystem/rag-pipeline/tests/test_coordination.py @@ -46,6 +46,33 @@ def test_project_mutation_lease_rejects_an_overlapping_job(): pass +def test_exact_generation_targets_have_independent_mutation_resources(): + coordinator = _coordinator() + + main = coordinator._resource_key("workspace", "project", "main-target") + develop = coordinator._resource_key("workspace", "project", "develop-target") + + assert main != develop + assert main == coordinator._resource_key("workspace", "project", "main-target") + + +def test_branch_publication_scope_serializes_only_the_same_branch_head(): + coordinator = _coordinator() + + main = coordinator._resource_key( + "workspace", "project", "main-target", "branch-head:main" + ) + main_next = coordinator._resource_key( + "workspace", "project", "next-main-target", "branch-head:main" + ) + develop = coordinator._resource_key( + "workspace", "project", "develop-target", "branch-head:develop" + ) + + assert main == main_next + assert main != develop + + def test_project_mutation_coordination_fails_closed_when_redis_is_unavailable(): coordinator = _coordinator() coordinator._client.set.side_effect = RuntimeError("redis unavailable") diff --git a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py new file mode 100644 index 000000000..d5ad8f361 --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py @@ -0,0 +1,258 @@ +from types import SimpleNamespace +from unittest.mock import MagicMock +import hashlib +import json +import pytest + +from llama_index.core.schema import TextNode +from qdrant_client import QdrantClient +from qdrant_client.models import ( + CreateAlias, + CreateAliasOperation, + Distance, + VectorParams, +) + +from rag_pipeline.core.generation_manifest import ( + build_generation_manifest_node, + collect_generation_members, + compute_generation_members_digest, +) +from rag_pipeline.core.index_manager.collection_manager import CollectionManager +from rag_pipeline.core.index_manager.manager import RAGIndexManager +from rag_pipeline.core.index_manager.point_operations import PointOperations +from rag_pipeline.core.index_manager.stats_manager import StatsManager +from rag_pipeline.core.revision_preflight import read_repository_revision_preflight +from rag_pipeline.core.repository_overlay import IncrementalIndexPreconditionError +from rag_pipeline.core.source_tree import compute_repository_source_tree_sha256 + + +SOURCE_COMMIT = "a" * 40 +TARGET_COMMIT = "b" * 40 +SOURCE_TREE = "c" * 64 + + +class _Embedding: + def get_text_embedding_batch(self, texts): + return [[1.0, 0.0, 0.0, 0.0] for _ in texts] + + +class _Lease: + token = "d" * 32 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def assert_owned(self): + return None + + +class _Coordinator: + def __init__(self): + self.calls = [] + + def acquire(self, *_args, **kwargs): + self.calls.append((tuple(_args), kwargs)) + return _Lease() + + +def _sealed_source(client, point_ops, collection): + identity = { + "plugin_ids": [], + "plugin_fingerprint": "sha256:" + "1" * 64, + "plugin_descriptor_fingerprint": "sha256:" + "2" * 64, + "plugin_implementation_fingerprint": "sha256:" + "3" * 64, + "index_representation_fingerprint": "sha256:" + "4" * 64, + } + facts = json.dumps( + { + "revision": SOURCE_COMMIT, + "paths": ["src/A.java", "src/B.java"], + "markerContents": {}, + }, + sort_keys=True, + separators=(",", ":"), + ) + nodes = [ + TextNode(text="alpha", metadata={ + "workspace": "ws", "project": "project", + "path": "src/A.java", "branch": "develop", + "commit": SOURCE_COMMIT, **identity, + }), + TextNode(text="beta", metadata={ + "workspace": "ws", "project": "project", + "path": "src/B.java", "branch": "develop", + "commit": SOURCE_COMMIT, **identity, + }), + TextNode(text=facts, metadata={ + "workspace": "ws", "project": "project", + "path": "__analysis_state__/repository-facts/000000.state", + "branch": "develop", "commit": SOURCE_COMMIT, + "repository_facts_state": True, + "facts_part": 0, + "facts_parts": 1, + "facts_content_sha256": hashlib.sha256( + facts.encode("utf-8") + ).hexdigest(), + **identity, + }), + ] + assert point_ops.process_and_upsert_chunks( + nodes, collection, "ws", "project", "develop" + ) == (3, 0) + members = collect_generation_members( + client, collection, "develop", SOURCE_COMMIT + ) + manifest = build_generation_manifest_node( + workspace="ws", + project="project", + branch="develop", + commit=SOURCE_COMMIT, + member_count=len(members), + members_sha256=compute_generation_members_digest(members), + source_tree_sha256=SOURCE_TREE, + index_include_patterns=(), + index_exclude_patterns=(), + identity_metadata=identity, + ) + assert point_ops.process_and_upsert_chunks( + [manifest], collection, "ws", "project", "develop" + ) == (1, 0) + + +def test_copy_on_write_advance_keeps_source_and_seals_target_revision(tmp_path): + client = QdrantClient(":memory:") + source_physical = "develop_source_physical" + source_alias = "develop_source" + target_alias = "develop_target" + client.create_collection( + collection_name=source_physical, + vectors_config=VectorParams(size=4, distance=Distance.COSINE), + ) + client.update_collection_aliases(change_aliases_operations=[ + CreateAliasOperation(create_alias=CreateAlias( + alias_name=source_alias, + collection_name=source_physical, + )) + ]) + point_ops = PointOperations( + client, _Embedding(), batch_size=50, embedding_dim=4 + ) + _sealed_source(client, point_ops, source_physical) + + manager = object.__new__(RAGIndexManager) + manager.qdrant_client = client + manager._collection_manager = CollectionManager(client, 4) + manager._mutation_coordinator = _Coordinator() + manager._file_ops = MagicMock() + manager._point_ops = point_ops + manager._stats_manager = StatsManager(client, "rag") + manager.config = SimpleNamespace(qdrant_collection_prefix="rag") + target_tree = compute_repository_source_tree_sha256(tmp_path) + + result = manager.advance_generation( + source_collection_target=source_alias, + target_collection_target=target_alias, + source_commit=SOURCE_COMMIT, + source_tree_sha256=target_tree, + updated_file_paths=[], + deleted_file_paths=[], + repo_base=str(tmp_path), + workspace="ws", + project="project", + branch="develop", + commit=TARGET_COMMIT, + publish_branch_alias=True, + ) + + source = read_repository_revision_preflight( + client, source_alias, "develop", SOURCE_COMMIT + ) + target = read_repository_revision_preflight( + client, target_alias, "develop", TARGET_COMMIT + ) + assert source is not None + assert target is not None + assert target["generation_manifest_sha256"] == ( + result.generation_manifest_sha256 + ) + assert target["generation_manifest_sha256"] != ( + source["generation_manifest_sha256"] + ) + assert result.collection_target == target_alias + aliases = { + alias.alias_name: alias.collection_name + for alias in client.get_aliases().aliases + } + assert aliases["rag_ws__project__develop"] == aliases[target_alias] + manager._file_ops.apply_changes.assert_called_once() + assert manager._mutation_coordinator.calls[-1][1]["publication_scope"] == ( + "branch-head:develop" + ) + + +def test_advance_is_idempotent_when_exact_target_already_exists(): + manager = object.__new__(RAGIndexManager) + manager._mutation_coordinator = _Coordinator() + manager._collection_manager = MagicMock() + manager._collection_manager.resolve_collection_target.side_effect = [ + "source-physical", "target-physical" + ] + manager.qdrant_client = MagicMock() + manager._stats_manager = MagicMock() + expected = SimpleNamespace(generation_manifest_sha256="e" * 64) + manager._stats_manager.get_branch_stats.return_value = expected + + import rag_pipeline.core.index_manager.manager as manager_module + original = manager_module.read_repository_revision_preflight + manager_module.read_repository_revision_preflight = ( + lambda *_args, **_kwargs: {"generation_manifest_sha256": "e" * 64} + ) + try: + result = manager.advance_generation( + "source", "target", SOURCE_COMMIT, SOURCE_TREE, [], [], None, + "ws", "project", "develop", TARGET_COMMIT, + ) + finally: + manager_module.read_repository_revision_preflight = original + + assert result is expected + manager._collection_manager.create_pending_collection.assert_not_called() + + +def test_exact_generation_delete_verifies_tenant_coordinates(): + client = QdrantClient(":memory:") + client.create_collection( + collection_name="foreign_generation", + vectors_config=VectorParams(size=4, distance=Distance.COSINE), + ) + point_ops = PointOperations(client, _Embedding(), embedding_dim=4) + point_ops.process_and_upsert_chunks( + [TextNode(text="secret", metadata={ + "workspace": "other-ws", + "project": "other-project", + "branch": "develop", + "path": "Secret.java", + })], + "foreign_generation", + "other-ws", + "other-project", + "develop", + ) + manager = object.__new__(RAGIndexManager) + manager.qdrant_client = client + manager._collection_manager = CollectionManager(client, 4) + manager._mutation_coordinator = _Coordinator() + + with pytest.raises( + IncrementalIndexPreconditionError, + match="does not belong", + ): + manager.delete_collection_target( + "ws", "project", "develop", "foreign_generation" + ) + + assert manager._collection_manager.collection_exists("foreign_generation") diff --git a/python-ecosystem/rag-pipeline/tests/test_index_manager.py b/python-ecosystem/rag-pipeline/tests/test_index_manager.py index d0598a248..fa1808cb2 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_manager.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_manager.py @@ -108,6 +108,24 @@ def test_create_pending_collection_uses_unique_names(self): assert first != second + def test_atomic_assign_aliases_replaces_all_requested_aliases_in_one_call(self): + cm = self._make() + old = MagicMock() + old.alias_name = "codecrow_ws__project" + old.collection_name = "old-primary" + cm.client.get_aliases.return_value.aliases = [old] + + cm.atomic_assign_aliases({ + "codecrow_ws__project": "new-generation", + "codecrow_ws__project__develop": "new-generation", + }) + + cm.client.update_collection_aliases.assert_called_once() + operations = cm.client.update_collection_aliases.call_args.kwargs[ + "change_aliases_operations" + ] + assert len(operations) == 3 + def test_payload_index_failure_does_not_skip_remaining_indexes(self): cm = self._make() cm.client.create_payload_index.side_effect = [ @@ -274,7 +292,7 @@ def test_embed_and_create_points(self): points = po.embed_and_create_points([("point-id-1", mock_chunk)]) assert len(points) == 1 assert points[0].id == "point-id-1" - assert points[0].vector == [0.1, 0.2, 0.3] + assert points[0].vector == pytest.approx([0.1, 0.2, 0.3]) # ───────────────────────────────────────────────────────────── @@ -342,6 +360,16 @@ def test_list_all_indices(self): # ───────────────────────────────────────────────────────────── class TestRAGIndexManager: + @pytest.fixture(autouse=True) + def avoid_network_tokenizer_download(self, monkeypatch): + # Constructing LlamaIndex's default SentenceSplitter may lazily fetch + # the tiktoken vocabulary. These manager unit tests mock embeddings and + # Qdrant, so they must remain hermetic as well. + from rag_pipeline.core.index_manager.manager import Settings + + monkeypatch.setattr(Settings, "_node_parser", MagicMock(), raising=False) + + def _mock_config(self): mock_config = MagicMock() mock_config.qdrant_url = "http://localhost:6333" @@ -386,6 +414,20 @@ def test_get_project_collection_name(self, MockQdrant, mock_info, mock_create): assert name.startswith("rag_") assert "workspace" in name + def test_branch_operator_alias_is_readable_and_branch_specific(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + manager.config = MagicMock(qdrant_collection_prefix="rag") + manager.config.qdrant_collection_prefix = "rag" + + assert manager._get_branch_operator_alias( + "Workspace", "Project", "develop" + ) == "rag_workspace__project__develop" + assert manager._get_branch_operator_alias( + "Workspace", "Project", "release/1.2" + ).startswith("rag_workspace__project__release_1_2_") + @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") @patch("rag_pipeline.core.index_manager.manager.QdrantClient") diff --git a/python-ecosystem/rag-pipeline/tests/test_indexer.py b/python-ecosystem/rag-pipeline/tests/test_indexer.py index f549eb8a5..3918d0e9c 100644 --- a/python-ecosystem/rag-pipeline/tests/test_indexer.py +++ b/python-ecosystem/rag-pipeline/tests/test_indexer.py @@ -475,7 +475,7 @@ def test_rejected_vector_point_is_skipped_and_valid_index_is_published(self): assert result.chunk_count == 1 assert result.skipped_chunk_count == 1 - coll_mgr.atomic_alias_swap.assert_called_once() + coll_mgr.atomic_assign_aliases.assert_called_once_with({"alias1": "pending"}) stats_mgr.store_metadata.assert_called_once() def test_repository_architecture_is_streamed_and_indexed_as_context(self, tmp_path): @@ -691,28 +691,30 @@ def test_normal_swap(self): config = _mock_config() coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - coll_mgr.collection_exists.return_value = False - coll_mgr.alias_exists.return_value = True - coll_mgr.resolve_alias.return_value = "active" + coll_mgr.read_alias_targets.return_value = {"alias1": "active"} indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - old_target = indexer._perform_atomic_swap("alias1", "pending", old_alias_exists=True) + old_targets = indexer._perform_atomic_swap( + "alias1", "pending", ["alias1"] + ) - coll_mgr.atomic_alias_swap.assert_called_once() - assert old_target == "active" + coll_mgr.atomic_assign_aliases.assert_called_once_with({"alias1": "pending"}) + assert old_targets == {"alias1": "active"} coll_mgr.delete_collection.assert_not_called() def test_first_activation_has_no_rollback_target(self): config = _mock_config() coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - coll_mgr.resolve_alias.return_value = None + coll_mgr.read_alias_targets.return_value = {"alias1": None} indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - old_target = indexer._perform_atomic_swap("alias1", "pending", old_alias_exists=False) + old_targets = indexer._perform_atomic_swap( + "alias1", "pending", ["alias1"] + ) - assert old_target is None - coll_mgr.atomic_alias_swap.assert_called_once_with("alias1", "pending", False) + assert old_targets == {"alias1": None} + coll_mgr.atomic_assign_aliases.assert_called_once_with({"alias1": "pending"}) def test_metadata_failure_rolls_back_before_pending_collection_is_deleted(self): config = _mock_config() @@ -725,6 +727,7 @@ def test_metadata_failure_rolls_back_before_pending_collection_is_deleted(self): coll_mgr.create_pending_collection.return_value = "pending" coll_mgr.alias_exists.return_value = True coll_mgr.resolve_alias.return_value = "active" + coll_mgr.read_alias_targets.return_value = {"alias1": "active"} point_ops.client.get_collection.return_value = SimpleNamespace(points_count=1) stats_mgr.store_metadata.side_effect = RuntimeError("metadata unavailable") @@ -732,9 +735,9 @@ def test_metadata_failure_rolls_back_before_pending_collection_is_deleted(self): with pytest.raises(RuntimeError, match="metadata unavailable"): indexer.index_repository("/repo", "ws", "proj", "main", "abc123", "alias1") - assert coll_mgr.atomic_alias_swap.call_args_list == [ - call("alias1", "pending", True), - call("alias1", "active", True), + assert coll_mgr.atomic_assign_aliases.call_args_list == [ + call({"alias1": "pending"}), + call({"alias1": "active"}), ] coll_mgr.delete_collection.assert_called_with("pending") diff --git a/python-ecosystem/rag-pipeline/tests/test_loader_extended.py b/python-ecosystem/rag-pipeline/tests/test_loader_extended.py index 9dcf05d10..3574e114f 100644 --- a/python-ecosystem/rag-pipeline/tests/test_loader_extended.py +++ b/python-ecosystem/rag-pipeline/tests/test_loader_extended.py @@ -301,7 +301,10 @@ def test_skips_generated_assets(self, loader, tmp_path): def test_read_error_skipped(self, loader, tmp_path): (tmp_path / "a.py").write_text("code") - with patch("pathlib.Path.read_text", side_effect=PermissionError("no")): + with patch( + "rag_pipeline.core.loader.read_repository_file_bytes", + side_effect=PermissionError("no"), + ): docs = loader.load_specific_files( [Path("a.py")], tmp_path, "ws", "proj", "main", "abc" ) diff --git a/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py b/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py new file mode 100644 index 000000000..d91ae1cff --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py @@ -0,0 +1,274 @@ +from types import SimpleNamespace + +import pytest +from llama_index.core.schema import TextNode +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams + +from rag_pipeline.core.index_manager.indexer import FileOperations +from rag_pipeline.core.index_manager.point_operations import PointOperations +from rag_pipeline.core.pr_overlay_manifest import read_pr_overlay_generation +from rag_pipeline.core.repository_overlay import ( + IncrementalIndexPreconditionError, +) + + +SOURCE_REVISION = "a" * 40 +BASE_REVISION = "b" * 40 +BASE_MANIFEST = "c" * 64 +GENERATION_FINGERPRINT = "sha256:" + "d" * 64 +OVERLAY_REPRESENTATION = "sha256:" + "e" * 64 +NEXT_SOURCE_REVISION = "1" * 40 +NEXT_GENERATION_FINGERPRINT = "sha256:" + "2" * 64 + + +def _operations(): + client = QdrantClient(":memory:") + client.create_collection( + collection_name="overlay", + vectors_config=VectorParams(size=2, distance=Distance.COSINE), + ) + embed_model = SimpleNamespace( + get_text_embedding_batch=lambda texts: [ + [0.1, 0.2] for _ in texts + ], + ) + point_operations = PointOperations( + client, + embed_model, + embedding_dim=2, + ) + file_operations = FileOperations( + client, + point_operations, + SimpleNamespace(), + SimpleNamespace(), + SimpleNamespace(), + SimpleNamespace(), + ) + return client, file_operations + + +def _node( + text="final class Service {}", + *, + source_revision=SOURCE_REVISION, + generation_fingerprint=GENERATION_FINGERPRINT, +): + return TextNode( + text=text, + metadata={ + "workspace": "workspace", + "project": "project", + "branch": "main", + "path": "app/code/Service.php", + "pr": True, + "pr_number": 42, + "pr_branch": "main", + "pr_source_revision": source_revision, + "pr_base_revision": BASE_REVISION, + "pr_base_generation_manifest_sha256": BASE_MANIFEST, + "pr_generation_fingerprint": generation_fingerprint, + "pr_overlay_representation_fingerprint": ( + OVERLAY_REPRESENTATION + ), + }, + ) + + +def _read( + client, + manifest_sha256=None, + *, + source_revision=SOURCE_REVISION, + generation_fingerprint=GENERATION_FINGERPRINT, +): + return read_pr_overlay_generation( + client, + "overlay", + workspace="workspace", + project="project", + pr_number=42, + branch="main", + base_branch="main", + source_revision=source_revision, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=generation_fingerprint, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + expected_manifest_sha256=manifest_sha256, + ) + + +def test_pr_overlay_seal_binds_exact_persisted_membership(): + client, operations = _operations() + + count, receipt = operations.replace_pr_overlay_generation( + [_node()], + [], + "overlay", + "workspace", + "project", + "__pr__/42/main", + pr_number=42, + branch="main", + base_branch="main", + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=GENERATION_FINGERPRINT, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + identity_metadata={ + "index_representation_fingerprint": "sha256:" + "f" * 64, + }, + ) + + assert count == 1 + assert receipt["overlay_generation_member_count"] == 1 + assert len(receipt["overlay_generation_manifest_sha256"]) == 64 + assert _read( + client, + receipt["overlay_generation_manifest_sha256"], + ) == receipt + + +def test_pr_overlay_rejects_tampered_member_with_same_count(): + client, operations = _operations() + _, receipt = operations.replace_pr_overlay_generation( + [_node()], + [], + "overlay", + "workspace", + "project", + "__pr__/42/main", + pr_number=42, + branch="main", + base_branch="main", + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=GENERATION_FINGERPRINT, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + identity_metadata={}, + ) + points, _ = client.scroll( + collection_name="overlay", + limit=10, + with_payload=True, + with_vectors=False, + ) + member = next( + point + for point in points + if not (point.payload or {}).get("pr_overlay_generation_manifest") + ) + client.set_payload( + collection_name="overlay", + payload={"text": "tampered"}, + points=[member.id], + ) + + with pytest.raises( + IncrementalIndexPreconditionError, + match="member integrity", + ): + _read(client, receipt["overlay_generation_manifest_sha256"]) + + +def test_pr_overlay_without_unique_manifest_is_not_reusable(): + client, operations = _operations() + chunk_data = operations.point_ops.prepare_chunks_for_embedding( + [_node()], + "workspace", + "project", + "__pr__/42/main", + ) + points = operations.point_ops.embed_and_create_points(chunk_data) + operations.point_ops.upsert_points("overlay", points) + operations.point_ops._seal_persisted_point_digests("overlay", points) + + with pytest.raises( + IncrementalIndexPreconditionError, + match="manifest is missing", + ): + _read(client) + + +def test_failed_new_generation_never_overwrites_or_hides_prior_lease(): + client, operations = _operations() + _, prior_receipt = operations.replace_pr_overlay_generation( + [_node()], + [], + "overlay", + "workspace", + "project", + f"__pr__/42/main/{GENERATION_FINGERPRINT}", + pr_number=42, + branch="main", + base_branch="main", + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=GENERATION_FINGERPRINT, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + identity_metadata={}, + ) + prior_points, _ = client.scroll( + collection_name="overlay", + limit=20, + with_payload=True, + with_vectors=True, + ) + + original_upsert = operations.point_ops.upsert_points + observed_prior_receipts = [] + + def fail_new_manifest(collection_name, points): + if any( + (point.payload or {}).get("pr_overlay_generation_manifest") + for point in points + ): + # The new generation's members are already persisted here. An + # exact reader of the prior lease must still see only prior IDs. + observed_prior_receipts.append( + _read( + client, + prior_receipt[ + "overlay_generation_manifest_sha256" + ], + ) + ) + return 0, len(points) + return original_upsert(collection_name, points) + + operations.point_ops.upsert_points = fail_new_manifest + with pytest.raises(RuntimeError, match="manifest write was incomplete"): + operations.replace_pr_overlay_generation( + [ + _node( + "final class ServiceV2 {}", + source_revision=NEXT_SOURCE_REVISION, + generation_fingerprint=NEXT_GENERATION_FINGERPRINT, + ), + ], + prior_points, + "overlay", + "workspace", + "project", + f"__pr__/42/main/{NEXT_GENERATION_FINGERPRINT}", + pr_number=42, + branch="main", + base_branch="main", + source_revision=NEXT_SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_MANIFEST, + generation_fingerprint=NEXT_GENERATION_FINGERPRINT, + overlay_representation_fingerprint=OVERLAY_REPRESENTATION, + identity_metadata={}, + ) + + assert observed_prior_receipts == [prior_receipt] + assert _read( + client, + prior_receipt["overlay_generation_manifest_sha256"], + ) == prior_receipt diff --git a/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py b/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py new file mode 100644 index 000000000..3bd6eb62b --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py @@ -0,0 +1,528 @@ +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from fastapi import HTTPException + +from rag_pipeline.api.models import ( + DeterministicContextRequest, + PRContextRequest, + QueryRequest, +) +from rag_pipeline.api.routers.query import ( + _query_pr_indexed_data, + get_deterministic_context, + get_pr_context, + semantic_search, +) +from rag_pipeline.core.repository_overlay import ( + IncrementalIndexPreconditionError, +) +from rag_pipeline.services.base import RAGQueryBase +from rag_pipeline.services.pr_context import PRContextMixin +from rag_pipeline.services.semantic_search import SemanticSearchMixin + + +BASE_REVISION = "a" * 40 +SOURCE_REVISION = "b" * 40 +BASE_GENERATION = "c" * 64 +PR_GENERATION = "sha256:" + "d" * 64 +PR_OVERLAY_MANIFEST = "e" * 64 +OVERLAY_REPRESENTATION = "sha256:" + "f" * 64 + + +def _overlay_receipt(manifest=PR_OVERLAY_MANIFEST): + return { + "overlay_generation_member_count": 1, + "overlay_generation_members_sha256": "1" * 64, + "overlay_generation_manifest_sha256": manifest, + } + + +def _manager(): + manager = MagicMock() + manager._get_project_collection_name.return_value = "rag_ws__project" + manager._collection_manager.collection_exists.return_value = True + manager._collection_manager.resolve_collection_target.return_value = ( + "rag_ws__project_active" + ) + manager.pr_overlay_representation_fingerprint = OVERLAY_REPRESENTATION + return manager + + +def test_semantic_query_detects_generation_swap_during_request(): + manager = _manager() + manager.get_revision_preflight.side_effect = [ + {"generation_manifest_sha256": BASE_GENERATION}, + {"generation_manifest_sha256": "e" * 64}, + ] + service = MagicMock() + service.semantic_search.return_value = [] + request = QueryRequest( + query="dependency lookup", + workspace="ws", + project="project", + branch="main", + repository_revision=BASE_REVISION, + repository_generation_manifest_sha256=BASE_GENERATION, + ) + + with patch( + "rag_pipeline.api.routers.query._get_singletons", + return_value=(manager, service), + ): + with pytest.raises(HTTPException) as exception: + semantic_search(request) + + assert exception.value.status_code == 409 + assert "generation changed" in exception.value.detail + service.semantic_search.assert_called_once_with( + query="dependency lookup", + workspace="ws", + project="project", + branch="main", + top_k=10, + filter_language=None, + expected_revision=BASE_REVISION, + collection_target="rag_ws__project_active", + ) + + +def test_pr_context_detects_base_generation_swap_after_retrieval(): + manager = _manager() + manager.get_revision_preflight.side_effect = [ + {"generation_manifest_sha256": BASE_GENERATION}, + {"generation_manifest_sha256": "e" * 64}, + ] + service = MagicMock() + service._collection_or_alias_exists.return_value = True + service.get_context_for_pr.return_value = { + "relevant_code": [], + "related_files": [], + "changed_files": ["src/Foo.php"], + } + request = PRContextRequest( + workspace="ws", + project="project", + branch="main", + base_branch="main", + changed_files=["src/Foo.php"], + pr_number=42, + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_GENERATION, + pr_generation_fingerprint=PR_GENERATION, + pr_overlay_generation_manifest_sha256=PR_OVERLAY_MANIFEST, + ) + + with ( + patch( + "rag_pipeline.api.routers.query._get_singletons", + return_value=(manager, service), + ), + patch( + "rag_pipeline.api.routers.query._query_pr_indexed_data", + return_value=[], + ) as query_overlay, + patch( + "rag_pipeline.api.routers.query.read_pr_overlay_generation", + return_value=_overlay_receipt(), + ), + ): + with pytest.raises(HTTPException) as exception: + get_pr_context(request) + + assert exception.value.status_code == 409 + assert "generation changed" in exception.value.detail + assert query_overlay.call_args.kwargs["collection_target"] == ( + "rag_ws__project_active" + ) + assert service.get_context_for_pr.call_args.kwargs[ + "collection_target" + ] == "rag_ws__project_active" + + +@pytest.mark.parametrize( + "probe_error", + [None, RuntimeError("qdrant unavailable")], + ids=["missing-collection", "backend-error"], +) +def test_revision_bound_pr_context_probe_failure_returns_http_conflict( + probe_error, +): + manager = _manager() + manager.get_revision_preflight.return_value = { + "generation_manifest_sha256": BASE_GENERATION, + } + + class ProbeFailingService(PRContextMixin, RAGQueryBase): + pass + + service = object.__new__(ProbeFailingService) + service.qdrant_client = MagicMock() + if probe_error is None: + service.qdrant_client.get_collections.return_value.collections = [] + service.qdrant_client.get_aliases.return_value.aliases = [] + else: + service.qdrant_client.get_collections.side_effect = probe_error + request = PRContextRequest( + workspace="ws", + project="project", + branch="main", + base_branch="main", + changed_files=["src/Foo.php"], + pr_number=42, + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_GENERATION, + pr_generation_fingerprint=PR_GENERATION, + pr_overlay_generation_manifest_sha256=PR_OVERLAY_MANIFEST, + ) + + with ( + patch( + "rag_pipeline.api.routers.query._get_singletons", + return_value=(manager, service), + ), + patch( + "rag_pipeline.api.routers.query._query_pr_indexed_data", + return_value=[], + ), + patch( + "rag_pipeline.api.routers.query.read_pr_overlay_generation", + return_value=_overlay_receipt(), + ), + ): + with pytest.raises(HTTPException) as exception: + get_pr_context(request) + + assert exception.value.status_code == 409 + assert ( + exception.value.detail + == "revision-bound PR-context collection is unavailable" + ) + manager.get_revision_preflight.assert_called_once() + + +def test_pr_overlay_query_filters_and_validates_exact_generation(): + manager = _manager() + point = SimpleNamespace( + payload={ + "path": "src/Foo.php", + "text": " Date: Mon, 10 Aug 2026 01:08:50 +0300 Subject: [PATCH 2/6] fix RAG generation safety, tenant isolation, and concurrent indexing progress --- .../java-shared/application.properties.sample | 10 +- deployment/config/rag-pipeline/.env.sample | 11 +- deployment/docker-compose.yml | 2 +- .../PullRequestAnalysisProcessor.java | 58 ++++-- .../PullRequestAnalysisProcessorTest.java | 49 +++++ .../rag/RagBranchIndexRepository.java | 12 ++ .../core/service/AnalysisJobService.java | 20 ++ .../codecrow/core/service/JobService.java | 26 ++- .../codecrow/core/service/JobServiceTest.java | 25 +++ ...BranchIndexBuildExecutorConfiguration.java | 4 +- .../BranchIndexGenerationBuildService.java | 40 +++- .../branch/BranchIndexMaintenanceService.java | 84 ++++++--- .../service/IncrementalRagUpdateService.java | 4 +- .../RagBranchIndexRegistryService.java | 46 +++-- .../service/RagOperationsServiceImpl.java | 43 ++++- ...BranchIndexGenerationBuildServiceTest.java | 51 +++++- .../RagBranchIndexRegistryServiceTest.java | 84 ++++++++- .../service/RagOperationsServiceImplTest.java | 17 +- .../generic/service/PipelineJobService.java | 20 ++ python-ecosystem/rag-pipeline/Dockerfile | 2 +- .../rag-pipeline/Dockerfile.observable | 2 +- python-ecosystem/rag-pipeline/main.py | 10 +- .../src/rag_pipeline/api/routers/index.py | 32 +++- .../src/rag_pipeline/api/routers/pr.py | 13 +- .../src/rag_pipeline/api/routers/query.py | 66 +++++-- .../core/index_manager/manager.py | 122 ++++++++++--- .../rag_pipeline/core/revision_preflight.py | 34 +++- .../core/revision_preflight_cache.py | 144 +++++++++++++++ .../src/rag_pipeline/models/config.py | 18 ++ .../services/deterministic_context.py | 18 ++ .../tests/test_generation_advance.py | 48 +++-- .../rag-pipeline/tests/test_index_manager.py | 18 ++ .../tests/test_revision_bound_queries.py | 69 +++++++ .../tests/test_revision_preflight.py | 41 +++++ .../tests/test_revision_preflight_cache.py | 171 ++++++++++++++++++ .../rag-pipeline/tests/test_router_pr.py | 48 +++++ 36 files changed, 1312 insertions(+), 150 deletions(-) create mode 100644 python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight_cache.py create mode 100644 python-ecosystem/rag-pipeline/tests/test_revision_preflight_cache.py diff --git a/deployment/config/java-shared/application.properties.sample b/deployment/config/java-shared/application.properties.sample index bbc17065c..837fa4a63 100644 --- a/deployment/config/java-shared/application.properties.sample +++ b/deployment/config/java-shared/application.properties.sample @@ -181,8 +181,14 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF #codecrow.rag.api.timeout.read=120 # RAG indexing timeout - 4 hours for large repositories #codecrow.rag.api.timeout.indexing=14400 -# Number of configured branch snapshots that an explicit Refresh all may build in parallel. -#codecrow.rag.branch-build.parallelism=4 +# Per-project fan-out for an explicit Refresh all. Two independent branch +# snapshots (for example main and develop) run in parallel without allowing a +# project with many retained branches to occupy all service slots. +#codecrow.rag.branch-build.parallelism=2 +# Dedicated service-wide capacity for full branch snapshot builds. This pool is +# separate from PR, branch-analysis, webhook and inference executors. Size it to +# the available RAG replicas and memory; it does not cap ordinary analyses. +#codecrow.rag.branch-build.global-parallelism=4 # Repair interval for readable Qdrant current-branch aliases of active generations. #codecrow.rag.operator-alias.reconcile-interval-ms=300000 #codecrow.rag.operator-alias.reconcile-initial-delay-ms=15000 diff --git a/deployment/config/rag-pipeline/.env.sample b/deployment/config/rag-pipeline/.env.sample index ca9b97ad6..8d6782902 100644 --- a/deployment/config/rag-pipeline/.env.sample +++ b/deployment/config/rag-pipeline/.env.sample @@ -55,10 +55,19 @@ SERVICE_SECRET=change-me-to-a-random-secret # === Queue and Server Runtime === # REDIS_URL=redis://redis:6379/1 # MAX_CONCURRENT_RAG_JOBS=2 -# UVICORN_WORKERS=4 +# Keep one API process by default: every worker loads its own embedding/indexing state. +# Increase only when the host has enough memory for another complete runtime. +# UVICORN_WORKERS=1 # Project mutation coordination is correctness-critical in multi-worker setups. # RAG_MUTATION_LEASE_SECONDS=300 # RAG_MUTATION_ACQUIRE_TIMEOUT_SECONDS=5 +# Exact-generation integrity scans are streamed, cached per immutable physical +# collection, and single-flight. This cap applies only to cold verification in +# one RAG service process; it does not cap indexing or application-wide reviews. +# RAG_REVISION_PREFLIGHT_CACHE_ENTRIES=512 +# 0 retains immutable positive receipts until LRU eviction or process restart. +# RAG_REVISION_PREFLIGHT_CACHE_TTL_SECONDS=0 +# RAG_REVISION_PREFLIGHT_MAX_CONCURRENCY=2 # Expired pending collections are retained for six hours by default. # RAG_PENDING_COLLECTION_MAX_AGE_SECONDS=21600 # RAG_PENDING_JANITOR_INTERVAL_SECONDS=3600 diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 1a0c5297f..cbe8434e9 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -278,7 +278,7 @@ services: SERVICE_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} QDRANT_API_KEY: ${QDRANT_API_KEY:?QDRANT_API_KEY must be set in .env} REDIS_URL: redis://redis:6379/1 - #UVICORN_WORKERS: 1 + UVICORN_WORKERS: ${RAG_UVICORN_WORKERS:-1} ports: - "127.0.0.1:${RAG_PIPELINE_HOST_PORT:-8004}:8001" #- "127.0.0.1:5678:5678" diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java index e6fea6caf..bb6cfc6b5 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java @@ -777,19 +777,8 @@ private void markPrCommitsAnalyzed( List newestFirst = client.getCommitHistory( workspace, repository, sourceBranch, SCM_EVIDENCE_COMMIT_LIMIT); - List prCommits = new java.util.ArrayList<>(); - for (VcsCommit commit : newestFirst == null - ? List.of() : newestFirst) { - if (targetBaseRevision != null - && targetBaseRevision.equals(commit.hash())) { - break; - } - prCommits.add(commit); - if (commitHash.equals(commit.hash()) - && targetBaseRevision == null) { - break; - } - } + List prCommits = selectPrEvidenceCommits( + newestFirst, commitHash, targetBaseRevision); Collections.reverse(prCommits); if (!prCommits.isEmpty()) { scmEvidenceService.capture( @@ -852,6 +841,49 @@ private void markPrCommitsAnalyzed( } } + /** + * Select only the ancestry that belongs to the revision actually reviewed. + * Provider history is newest-first and may already contain commits pushed + * after the webhook event, so collection cannot begin until the requested + * PR head is reached. + */ + static List selectPrEvidenceCommits( + List newestFirst, + String requestedRevision, + String targetBaseRevision) { + if (newestFirst == null || newestFirst.isEmpty() + || requestedRevision == null || requestedRevision.isBlank()) { + return new java.util.ArrayList<>(); + } + List selected = new java.util.ArrayList<>(); + boolean requestedRevisionReached = false; + boolean baseRevisionReached = targetBaseRevision == null; + for (VcsCommit commit : newestFirst) { + if (!requestedRevisionReached) { + if (!requestedRevision.equals(commit.hash())) { + continue; + } + requestedRevisionReached = true; + } + if (targetBaseRevision != null + && targetBaseRevision.equals(commit.hash())) { + baseRevisionReached = true; + break; + } + selected.add(commit); + if (targetBaseRevision == null) { + break; + } + } + if (!baseRevisionReached && selected.size() > 1) { + // The bounded provider window did not prove where PR ancestry ends. + // Retain the reviewed head receipt, but do not claim older commits + // that may predate the PR base. + return new java.util.ArrayList<>(selected.subList(0, 1)); + } + return selected; + } + /** * Ensures RAG index is up-to-date for the PR target branch. *

diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java index 5560b5400..13668ba2c 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java @@ -30,6 +30,7 @@ import org.rostilos.codecrow.core.service.CodeAnalysisService; import org.rostilos.codecrow.core.service.TaskImplementationEvidenceService; import org.rostilos.codecrow.filecontent.service.FileSnapshotService; +import org.rostilos.codecrow.vcsclient.model.VcsCommit; import org.rostilos.codecrow.analysisengine.service.AstScopeEnricher; import org.rostilos.codecrow.analysisengine.service.pr.PrIssueTrackingService; import org.rostilos.codecrow.analysisengine.util.PromptDryRunMode; @@ -152,6 +153,54 @@ private PrProcessRequest createRequest() { return request; } + @Test + @DisplayName("SCM evidence starts at the reviewed PR revision") + void scmEvidenceExcludesCommitsPushedAfterReviewedHead() { + List selected = PullRequestAnalysisProcessor.selectPrEvidenceCommits( + List.of( + commit("newer-2"), + commit("newer-1"), + commit("reviewed-head"), + commit("pr-parent"), + commit("target-base"), + commit("older")), + "reviewed-head", + "target-base"); + + assertThat(selected).extracting(VcsCommit::hash) + .containsExactly("reviewed-head", "pr-parent"); + } + + @Test + @DisplayName("SCM evidence stays empty when reviewed head is outside provider history") + void scmEvidenceDoesNotClaimUnknownHistoryWindow() { + List selected = PullRequestAnalysisProcessor.selectPrEvidenceCommits( + List.of(commit("newer"), commit("target-base")), + "reviewed-head", + "target-base"); + + assertThat(selected).isEmpty(); + } + + @Test + @DisplayName("SCM evidence keeps only reviewed head when PR base is outside provider history") + void scmEvidenceDoesNotClaimCommitsPastUnknownBase() { + List selected = PullRequestAnalysisProcessor.selectPrEvidenceCommits( + List.of( + commit("reviewed-head"), + commit("unknown-parent"), + commit("older")), + "reviewed-head", + "target-base"); + + assertThat(selected).extracting(VcsCommit::hash) + .containsExactly("reviewed-head"); + } + + private static VcsCommit commit(String hash) { + return new VcsCommit(hash, hash, "author", "author@example.test", null, List.of()); + } + @Nested @DisplayName("process()") class ProcessTests { diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java index 7fa4cd5d4..5660ee453 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java @@ -1,8 +1,10 @@ package org.rostilos.codecrow.core.persistence.repository.rag; +import jakarta.persistence.LockModeType; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -19,6 +21,16 @@ public interface RagBranchIndexRepository extends JpaRepository findByProjectIdAndBranchName(Long projectId, String branchName); + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT b FROM RagBranchIndex b WHERE b.project.id = :projectId AND b.branchName = :branchName") + Optional findByProjectIdAndBranchNameForUpdate( + @Param("projectId") Long projectId, + @Param("branchName") String branchName); + + @Lock(LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT b FROM RagBranchIndex b WHERE b.id = :id") + Optional findByIdForPublication(@Param("id") Long id); + List findByProjectId(Long projectId); List findByIndexKind(RagBranchIndexKind indexKind); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java index 4bfc6618e..043b4fbe2 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java @@ -32,6 +32,26 @@ public interface AnalysisJobService { */ Job createRagIndexJob(Project project, boolean isInitial, JobTriggerSource triggerSource); + /** + * Create a branch-bound RAG indexing job. Hosts that persist jobs should + * override this method so the branch and revision are durable before the + * worker starts; the default keeps non-persistent host implementations + * source-compatible. + */ + default Job createRagIndexJob( + Project project, + boolean isInitial, + JobTriggerSource triggerSource, + String branchName, + String commitHash) { + Job job = createRagIndexJob(project, isInitial, triggerSource); + if (job != null) { + job.setBranchName(branchName); + job.setCommitHash(commitHash); + } + return job; + } + /** * Start a job. * @param job The job to start diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java index 23190f904..5382c0510 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java @@ -198,17 +198,39 @@ public Job createRagIndexJob( boolean isInitial, JobTriggerSource triggerSource, User triggeredBy + ) { + return createRagIndexJob( + project, isInitial, triggerSource, triggeredBy, null, null); + } + + /** Create a durable RAG job bound to the branch revision it builds. */ + @Transactional + public Job createRagIndexJob( + Project project, + boolean isInitial, + JobTriggerSource triggerSource, + User triggeredBy, + String branchName, + String commitHash ) { Job job = new Job(); job.setProject(project); job.setJobType(isInitial ? JobType.RAG_INITIAL_INDEX : JobType.RAG_INCREMENTAL_INDEX); job.setTriggerSource(triggerSource); job.setTriggeredBy(triggeredBy); - job.setTitle(isInitial ? "Initial RAG Indexing" : "Incremental RAG Update"); + job.setBranchName(branchName); + job.setCommitHash(commitHash); + String operation = isInitial ? "Initial RAG Indexing" : "Incremental RAG Update"; + job.setTitle(branchName == null || branchName.isBlank() + ? operation + : operation + ": " + branchName); job.setStatus(JobStatus.PENDING); job = jobRepository.save(job); - addLog(job, JobLogLevel.INFO, "init", "RAG indexing job created"); + addLog(job, JobLogLevel.INFO, "init", + branchName == null || branchName.isBlank() + ? "RAG indexing job created" + : "RAG indexing job created for branch: " + branchName); return job; } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java index 1c717b3b8..d7b46d876 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java @@ -190,6 +190,31 @@ void shouldCreateIncrementalRagIndexJob() { assertThat(job.getJobType()).isEqualTo(JobType.RAG_INCREMENTAL_INDEX); assertThat(job.getTitle()).contains("Incremental"); } + + @Test + @DisplayName("should persist branch and revision for a branch RAG job") + void shouldCreateBranchBoundRagIndexJob() { + Project project = createProject(1L, "Test"); + + when(jobRepository.save(any(Job.class))).thenAnswer(inv -> { + Job j = inv.getArgument(0); + setField(j, "id", 104L); + return j; + }); + when(jobLogRepository.save(any(JobLog.class))).thenAnswer(inv -> inv.getArgument(0)); + + Job job = jobService.createRagIndexJob( + project, + false, + JobTriggerSource.UI, + null, + "develop", + "abc123"); + + assertThat(job.getBranchName()).isEqualTo("develop"); + assertThat(job.getCommitHash()).isEqualTo("abc123"); + assertThat(job.getTitle()).isEqualTo("Incremental RAG Update: develop"); + } } @Nested diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java index b5627652b..b014abc3f 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java @@ -7,13 +7,13 @@ import java.util.concurrent.Executor; -/** Dedicated bounded concurrency for independent configured branch snapshots. */ +/** Dedicated service capacity for independent configured branch snapshots. */ @Configuration public class BranchIndexBuildExecutorConfiguration { @Bean(name = "branchIndexBuildExecutor") public Executor branchIndexBuildExecutor( - @Value("${codecrow.rag.branch-build.parallelism:4}") int parallelism) { + @Value("${codecrow.rag.branch-build.global-parallelism:4}") int parallelism) { int workers = Math.max(1, parallelism); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(workers); diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java index 04cbb8351..fa465d3d8 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java @@ -3,10 +3,13 @@ import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import java.io.IOException; @@ -29,6 +32,8 @@ */ @Service public class BranchIndexGenerationBuildService { + private static final Logger log = LoggerFactory.getLogger( + BranchIndexGenerationBuildService.class); private static final long HEARTBEAT_INTERVAL_SECONDS = 15; private final BranchArchiveService archiveService; @@ -161,22 +166,25 @@ private Map buildInternal( snapshot.toString(), project.getWorkspace().getName(), project.getNamespace(), branch, revision, includePatterns, excludePatterns, registration.generation().getCollectionName(), - publishBranchAlias, publishLegacyProjectAlias) + false, false) : pipelineClient.indexRepository( snapshot.toString(), project.getWorkspace().getName(), project.getNamespace(), branch, revision, includePatterns, excludePatterns, registration.generation().getCollectionName(), - publishBranchAlias, publishLegacyProjectAlias, + false, false, progressEvents); Object manifest = result.get("generation_manifest_sha256"); if (!(manifest instanceof String digest) || digest.isBlank()) { throw new IOException("RAG full branch generation has no manifest digest"); } - registryService.publish( + var published = registryService.publish( registration.operation().getId(), digest, number(result.get("document_count")), number(result.get("chunk_count"))); + publishReadableAliasesIfActive( + project, branch, revision, registration.generation().getCollectionName(), + published, publishBranchAlias, publishLegacyProjectAlias); return result; } catch (Throwable failure) { registryService.fail( @@ -199,6 +207,32 @@ private Map buildInternal( } } + private void publishReadableAliasesIfActive( + Project project, + String branch, + String revision, + String collectionTarget, + org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration published, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias) { + if (published == null + || published.getStatus() != RagBranchIndexGenerationStatus.ACTIVE + || !publishBranchAlias) { + return; + } + try { + pipelineClient.publishGenerationAliases( + project.getWorkspace().getName(), project.getNamespace(), + branch, revision, collectionTarget, + true, publishLegacyProjectAlias); + } catch (IOException aliasFailure) { + // Readable aliases are operator convenience. Exact retrieval uses + // the registry target, and reconciliation repairs this alias later. + log.warn("Could not publish readable aliases for RAG generation {}: {}", + published.getId(), aliasFailure.getMessage()); + } + } + private static int number(Object value) { return value instanceof Number number ? number.intValue() : 0; } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java index b8c4cee1b..d5813aedd 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java @@ -4,6 +4,7 @@ import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobLogLevel; import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; @@ -13,6 +14,7 @@ import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -44,6 +46,7 @@ public class BranchIndexMaintenanceService { private final AnalysisLockService lockService; private final AnalysisJobService jobService; private final Executor branchIndexBuildExecutor; + private final int perProjectParallelism; public BranchIndexMaintenanceService( RagOperationsService ragOperationsService, @@ -52,7 +55,8 @@ public BranchIndexMaintenanceService( RagIndexTrackingService trackingService, AnalysisLockService lockService, AnalysisJobService jobService, - @Qualifier("branchIndexBuildExecutor") Executor branchIndexBuildExecutor) { + @Qualifier("branchIndexBuildExecutor") Executor branchIndexBuildExecutor, + @Value("${codecrow.rag.branch-build.parallelism:2}") int perProjectParallelism) { this.ragOperationsService = ragOperationsService; this.vcsClientProvider = vcsClientProvider; this.generationBuildService = generationBuildService; @@ -60,6 +64,7 @@ public BranchIndexMaintenanceService( this.lockService = lockService; this.jobService = jobService; this.branchIndexBuildExecutor = branchIndexBuildExecutor; + this.perProjectParallelism = Math.max(1, perProjectParallelism); } public Map rebuild(Project project, String requestedBranch, boolean allConfiguredBranches, @@ -67,7 +72,6 @@ public Map rebuild(Project project, String requestedBranch, bool List branches = resolveBranches(project, requestedBranch, allConfiguredBranches); List completed = new ArrayList<>(); Map failures = new LinkedHashMap<>(); - Map> builds = new LinkedHashMap<>(); // Obtaining a provider client can refresh a shared installation token. Do // that small VCS preparation phase once at a time, then let the expensive @@ -87,28 +91,36 @@ public Map rebuild(Project project, String requestedBranch, bool "message", "RAG snapshot failed for branch '" + branch + "': " + message)); } } - for (BranchBuildPlan plan : plans) { - String branch = plan.branch(); - builds.put(branch, CompletableFuture.runAsync(() -> { - events.accept(Map.of("type", "progress", "stage", "branch", - "branch", branch, - "message", "Building exact RAG snapshot for branch '" + branch + "'")); - rebuildOne(project, plan, events); - }, branchIndexBuildExecutor)); - } - for (Map.Entry> build : builds.entrySet()) { - try { - build.getValue().join(); - completed.add(build.getKey()); - } catch (CompletionException failure) { - Throwable cause = failure.getCause() != null ? failure.getCause() : failure; - String message = cause.getMessage() != null - ? cause.getMessage() : cause.getClass().getSimpleName(); - failures.put(build.getKey(), message); - events.accept(Map.of("type", "progress", "stage", "branch_failed", - "branch", build.getKey(), - "message", "RAG snapshot failed for branch '" + build.getKey() - + "': " + message)); + // Limit one project's fan-out independently from the service capacity. + // A project with many retained branches can therefore use at most its + // configured share while builds for other tenants still occupy the + // remaining dedicated RAG slots. Each wave is fully parallel. + for (int from = 0; from < plans.size(); from += perProjectParallelism) { + int to = Math.min(plans.size(), from + perProjectParallelism); + Map> wave = new LinkedHashMap<>(); + for (BranchBuildPlan plan : plans.subList(from, to)) { + String branch = plan.branch(); + wave.put(branch, CompletableFuture.runAsync(() -> { + events.accept(Map.of("type", "progress", "stage", "branch", + "branch", branch, + "message", "Building exact RAG snapshot for branch '" + branch + "'")); + rebuildOne(project, plan, events); + }, branchIndexBuildExecutor)); + } + for (Map.Entry> build : wave.entrySet()) { + try { + build.getValue().join(); + completed.add(build.getKey()); + } catch (CompletionException failure) { + Throwable cause = failure.getCause() != null ? failure.getCause() : failure; + String message = cause.getMessage() != null + ? cause.getMessage() : cause.getClass().getSimpleName(); + failures.put(build.getKey(), message); + events.accept(Map.of("type", "progress", "stage", "branch_failed", + "branch", build.getKey(), + "message", "RAG snapshot failed for branch '" + build.getKey() + + "': " + message)); + } } } if (completed.isEmpty()) { @@ -161,9 +173,19 @@ private void rebuildOne(Project project, BranchBuildPlan plan, Consumer forwarded = new LinkedHashMap<>(event); forwarded.put("type", "progress"); forwarded.put("branch", branch); + String stage = String.valueOf( + forwarded.getOrDefault("stage", "indexing")); + String message = String.valueOf( + forwarded.getOrDefault("message", "Indexing branch '" + branch + "'")); + jobService.logToJob( + job, + JobLogLevel.INFO, + stage, + message, + forwarded); events.accept(forwarded); }); if (primary) { diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java index a51873b5d..e9e0f52e6 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java @@ -123,8 +123,8 @@ public Map performIncrementalUpdate( } /** - * Advances an immutable generation and, for retained branches, publishes - * operator aliases in the same RAG-side activation transaction. + * Advances an immutable generation. The registry owner publishes readable + * aliases only after it has accepted this generation as the current head. */ public Map performIncrementalUpdate( Project project, diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java index d6c4a9268..5b2a3ab80 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java @@ -71,7 +71,7 @@ public BuildRegistration registerBuild( } RagBranchIndex branchIndex = branchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branch) + .findByProjectIdAndBranchNameForUpdate(project.getId(), branch) .orElseGet(() -> new RagBranchIndex(project, branch, kind)); if (branchIndex.getIndexKind() == RagBranchIndexKind.LEGACY || kind == RagBranchIndexKind.PRIMARY @@ -110,10 +110,6 @@ public void startBuild(long operationId, Long jobId) { if (operation.getStatus() == RagIndexOperationStatus.FAILED) { operation.getGeneration().retry(); generationRepository.save(operation.getGeneration()); - operation.getGeneration().getBranchIndex().requestRevision( - operation.getToRevision()); - branchIndexRepository.save( - operation.getGeneration().getBranchIndex()); } operation.setJobId(jobId); operation.start(); @@ -135,9 +131,28 @@ public RagBranchIndexGeneration publish( throw new IllegalStateException("Only a building generation can be published"); } - RagBranchIndex branchIndex = generation.getBranchIndex(); + Long branchIndexId = generation.getBranchIndex().getId(); + RagBranchIndex branchIndex = branchIndexRepository + .findByIdForPublication(branchIndexId) + .orElseThrow(() -> new IllegalStateException( + "RAG branch index not found: " + branchIndexId)); + generation.setBranchIndex(branchIndex); + + String digest = requireText(manifestDigest, "manifestDigest"); + if (!generation.getRevision().equals(branchIndex.getDesiredCommitHash())) { + // The physical generation is complete and remains useful for exact + // revision reads, but a newer request owns the branch head. Record + // this operation as successful without regressing the active head. + generation.activate(digest, fileCount, chunkCount); + generation.supersede(); + generationRepository.save(generation); + operation.succeed(generation); + operationRepository.save(operation); + return generation; + } + RagBranchIndexGeneration previous = branchIndex.getActiveGeneration(); - generation.activate(requireText(manifestDigest, "manifestDigest"), fileCount, chunkCount); + generation.activate(digest, fileCount, chunkCount); generationRepository.save(generation); if (previous != null && previous.getId() != null && !previous.getId().equals(generation.getId())) { previous.supersede(); @@ -157,10 +172,19 @@ public void fail(long operationId, String errorMessage) { return; } String failure = requireText(errorMessage, "errorMessage"); - operation.getGeneration().fail(failure); - generationRepository.save(operation.getGeneration()); - operation.getGeneration().getBranchIndex().failUpdate(failure); - branchIndexRepository.save(operation.getGeneration().getBranchIndex()); + RagBranchIndexGeneration generation = operation.getGeneration(); + Long branchIndexId = generation.getBranchIndex().getId(); + RagBranchIndex branchIndex = branchIndexRepository + .findByIdForPublication(branchIndexId) + .orElseThrow(() -> new IllegalStateException( + "RAG branch index not found: " + branchIndexId)); + generation.setBranchIndex(branchIndex); + generation.fail(failure); + generationRepository.save(generation); + if (generation.getRevision().equals(branchIndex.getDesiredCommitHash())) { + branchIndex.failUpdate(failure); + branchIndexRepository.save(branchIndex); + } operation.fail(failure); operationRepository.save(operation); } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java index bf71b9862..a6a53ea8a 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java @@ -8,6 +8,7 @@ import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; @@ -307,8 +308,8 @@ public boolean triggerIncrementalUpdate( sourceGeneration.getRevision(), sourceGeneration.getCollectionName(), branchBuild.generation().getCollectionName(), - publishBranchAlias, - publishLegacyProjectAlias); + false, + false); } else { result = incrementalRagUpdateService.performIncrementalUpdate( project, @@ -328,11 +329,16 @@ public boolean triggerIncrementalUpdate( throw new IllegalStateException( "Advanced RAG generation has no manifest digest"); } - branchIndexRegistryService.publish( + RagBranchIndexGeneration published = branchIndexRegistryService.publish( branchBuild.operation().getId(), manifestDigest, ((Number) result.getOrDefault("document_count", 0)).intValue(), ((Number) result.getOrDefault("chunk_count", 0)).intValue()); + publishReadableAliasesIfActive( + project, branchName, commitHash, + branchBuild.generation().getCollectionName(), + published, publishBranchAlias, + publishLegacyProjectAlias); } } catch (Exception generationFailure) { if (branchBuild != null) { @@ -419,6 +425,30 @@ public boolean triggerIncrementalUpdate( } } + private void publishReadableAliasesIfActive( + Project project, + String branch, + String revision, + String collectionTarget, + RagBranchIndexGeneration published, + boolean publishBranchAlias, + boolean publishLegacyProjectAlias) { + if (published == null + || published.getStatus() != RagBranchIndexGenerationStatus.ACTIVE + || !publishBranchAlias) { + return; + } + try { + ragPipelineClient.publishGenerationAliases( + project.getWorkspace().getName(), project.getNamespace(), + branch, revision, collectionTarget, + true, publishLegacyProjectAlias); + } catch (IOException aliasFailure) { + log.warn("Readable alias publication failed for active RAG generation {}: {}", + published.getId(), aliasFailure.getMessage()); + } + } + /** * Rebuilds the effective range from the last completed RAG checkpoint. * The caller's branch-analysis diff is used only when no completed @@ -626,6 +656,13 @@ public boolean updateBranchIndex( eventConsumer.accept(Map.of( "type", "info", "message", String.format("Branch '%s' has same content as '%s'", targetBranch, baseBranch))); + if (usesExactGenerations(project)) { + // No completed checkpoint means there is still no exact + // target-branch generation. An empty tree delta must seed + // the complete revision rather than report a false success. + return triggerIncrementalUpdate( + project, targetBranch, targetCommit, "", eventConsumer); + } return true; } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java index 92515dd42..e9e14f200 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java @@ -70,11 +70,16 @@ void buildsPinnedSnapshotPublishesManifestAndRemovesTemporaryTree() throws Excep anyString(), eq("workspace"), eq("namespace"), eq("develop"), eq("develop-400"), eq(List.of("src/**")), eq(List.of("vendor/**")), eq("opaque-generation-target"), - eq(true), eq(false))) + eq(false), eq(false))) .thenReturn(Map.of( "generation_manifest_sha256", "manifest-400", "document_count", 231, "chunk_count", 400)); + when(registryService.publish(30L, "manifest-400", 231, 400)) + .thenAnswer(invocation -> { + generation.activate("manifest-400", 231, 400); + return generation; + }); ReflectionTestUtils.setField(project, "namespace", "namespace"); var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); ReflectionTestUtils.setField(workspace, "name", "workspace"); @@ -89,6 +94,9 @@ project, new VcsConnection(), "provider-workspace", "repo", "generation_manifest_sha256", "manifest-400"); verify(registryService).startBuild(30L, 77L); verify(registryService).publish(30L, "manifest-400", 231, 400); + verify(pipelineClient).publishGenerationAliases( + "workspace", "namespace", "develop", "develop-400", + "opaque-generation-target", true, false); ArgumentCaptor snapshot = ArgumentCaptor.forClass(Path.class); verify(archiveService).downloadAndExtractSnapshotToDirectory( any(), eq("provider-workspace"), eq("repo"), @@ -153,11 +161,16 @@ void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws when(pipelineClient.indexRepository( anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), anyList(), anyList(), eq("opaque-generation-target"), - eq(true), eq(false), any())) + eq(false), eq(false), any())) .thenReturn(Map.of( "generation_manifest_sha256", "fresh-manifest", "document_count", 231, "chunk_count", 400)); + when(registryService.publish(30L, "fresh-manifest", 231, 400)) + .thenAnswer(invocation -> { + generation.activate("fresh-manifest", 231, 400); + return generation; + }); var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); ReflectionTestUtils.setField(workspace, "name", "workspace"); project.setWorkspace(workspace); @@ -170,5 +183,39 @@ void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws verify(archiveService).downloadAndExtractSnapshotToDirectory( any(), eq("provider-workspace"), eq("repo"), eq("develop-400"), isNull(), any()); verify(registryService).publish(30L, "fresh-manifest", 231, 400); + verify(pipelineClient).publishGenerationAliases( + "workspace", "namespace", "develop", "develop-400", + "opaque-generation-target", true, false); + } + + @Test + void staleCompletedGenerationDoesNotPublishReadableAliases() throws Exception { + when(registryService.registerBuild(any(), anyString(), any(), isNull(), + anyString(), isNull())) + .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( + generation.getBranchIndex(), generation, operation, false)); + when(pipelineClient.indexRepository( + anyString(), anyString(), anyString(), anyString(), anyString(), + anyList(), anyList(), anyString(), eq(false), eq(false))) + .thenReturn(Map.of( + "generation_manifest_sha256", "manifest-400", + "document_count", 231, + "chunk_count", 400)); + generation.activate("manifest-400", 231, 400); + generation.supersede(); + when(registryService.publish(30L, "manifest-400", 231, 400)) + .thenReturn(generation); + var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + project.setNamespace("namespace"); + + service.build(project, new VcsConnection(), "provider-workspace", "repo", + "develop", "develop-400", RagBranchIndexKind.DURABLE, + List.of(), List.of()); + + verify(pipelineClient, never()).publishGenerationAliases( + anyString(), anyString(), anyString(), anyString(), anyString(), + anyBoolean(), anyBoolean()); } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java index 4adc89b69..7b5b30520 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java @@ -37,7 +37,7 @@ void setUp() { branchIndexRepository, generationRepository, operationRepository); project = new Project(); ReflectionTestUtils.setField(project, "id", 42L); - when(branchIndexRepository.save(any())).thenAnswer(invocation -> { + lenient().when(branchIndexRepository.save(any())).thenAnswer(invocation -> { RagBranchIndex value = invocation.getArgument(0); if (value.getId() == null) { value.setId(10L); @@ -64,7 +64,7 @@ void setUp() { void registersIdempotentTenantScopedBuildWithoutLeakingBranchName() { when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) .thenReturn(Optional.empty()); - when(branchIndexRepository.findByProjectIdAndBranchName(42L, "client/private-develop")) + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate(42L, "client/private-develop")) .thenReturn(Optional.empty()); var registration = service.registerBuild( @@ -95,7 +95,7 @@ void publishesNewGenerationAndSupersedesPreviousOneAtomically() { previous.setId(19L); previous.activate("manifest-400", 500, 1500); branchIndex.activate(previous); - when(branchIndexRepository.findByProjectIdAndBranchName(42L, "develop")) + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate(42L, "develop")) .thenReturn(Optional.of(branchIndex)); when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) .thenReturn(Optional.empty()); @@ -104,6 +104,8 @@ void publishesNewGenerationAndSupersedesPreviousOneAtomically() { project, "develop", RagBranchIndexKind.DURABLE, "develop-400", "develop-401", "representation"); when(operationRepository.findById(30L)).thenReturn(Optional.of(registration.operation())); + when(branchIndexRepository.findByIdForPublication(10L)) + .thenReturn(Optional.of(branchIndex)); service.startBuild(30L, 99L); RagBranchIndexGeneration published = service.publish(30L, "manifest-401", 501, 1504); @@ -117,6 +119,45 @@ void publishesNewGenerationAndSupersedesPreviousOneAtomically() { assertThat(registration.operation().getJobId()).isEqualTo(99L); } + @Test + void completedOlderGenerationDoesNotRegressNewerDesiredRevision() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + RagBranchIndexGeneration active = new RagBranchIndexGeneration( + branchIndex, "develop-400", "generation-400", null, + "develop-399", "representation"); + active.setId(19L); + active.activate("manifest-400", 500, 1500); + branchIndex.activate(active); + branchIndex.requestRevision("develop-402"); + + RagBranchIndexGeneration late = new RagBranchIndexGeneration( + branchIndex, "develop-401", "generation-401", active, + "develop-400", "representation"); + late.setId(20L); + RagIndexOperation operation = new RagIndexOperation( + project, "develop", "develop-400", "develop-401", "late-key"); + operation.setId(30L); + operation.setGeneration(late); + operation.start(); + when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + when(branchIndexRepository.findByIdForPublication(10L)) + .thenReturn(Optional.of(branchIndex)); + + RagBranchIndexGeneration published = service.publish( + 30L, "manifest-401", 501, 1504); + + assertThat(published.getStatus()) + .isEqualTo(RagBranchIndexGenerationStatus.SUPERSEDED); + assertThat(published.getManifestDigest()).isEqualTo("manifest-401"); + assertThat(branchIndex.getActiveGeneration()).isSameAs(active); + assertThat(branchIndex.getCommitHash()).isEqualTo("develop-400"); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-402"); + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); + verify(branchIndexRepository, never()).save(branchIndex); + } + @Test void failedReplacementKeepsLastVerifiedGenerationAvailable() { RagBranchIndex branchIndex = new RagBranchIndex(project, "master", RagBranchIndexKind.PRIMARY); @@ -127,7 +168,7 @@ void failedReplacementKeepsLastVerifiedGenerationAvailable() { active.setId(19L); active.activate("manifest-100", 500, 1500); branchIndex.activate(active); - when(branchIndexRepository.findByProjectIdAndBranchName(42L, "master")) + when(branchIndexRepository.findByProjectIdAndBranchNameForUpdate(42L, "master")) .thenReturn(Optional.of(branchIndex)); when(operationRepository.findByProjectIdAndOperationKey(eq(42L), anyString())) .thenReturn(Optional.empty()); @@ -136,6 +177,8 @@ void failedReplacementKeepsLastVerifiedGenerationAvailable() { project, "master", RagBranchIndexKind.PRIMARY, "master-100", "master-101", "representation"); when(operationRepository.findById(30L)).thenReturn(Optional.of(registration.operation())); + when(branchIndexRepository.findByIdForPublication(10L)) + .thenReturn(Optional.of(branchIndex)); service.fail(30L, "vector publication failed"); @@ -152,6 +195,7 @@ void failedIdempotentOperationCanRetryWithoutCreatingDuplicateGeneration() { RagBranchIndex branchIndex = new RagBranchIndex( project, "develop", RagBranchIndexKind.DURABLE); branchIndex.setId(10L); + branchIndex.requestRevision("develop-401"); RagBranchIndexGeneration generation = new RagBranchIndexGeneration( branchIndex, "develop-401", "generation-401", null, "develop-400", "representation"); @@ -173,7 +217,37 @@ void failedIdempotentOperationCanRetryWithoutCreatingDuplicateGeneration() { .isEqualTo(RagBranchIndexGenerationStatus.BUILDING); assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-401"); verify(generationRepository).save(generation); - verify(branchIndexRepository).save(branchIndex); + verify(branchIndexRepository, never()).save(branchIndex); verify(operationRepository).save(operation); } + + @Test + void lateFailureDoesNotOverwriteNewerDesiredRevisionState() { + RagBranchIndex branchIndex = new RagBranchIndex( + project, "develop", RagBranchIndexKind.DURABLE); + branchIndex.setId(10L); + branchIndex.requestRevision("develop-402"); + RagBranchIndexGeneration late = new RagBranchIndexGeneration( + branchIndex, "develop-401", "generation-401", null, + "develop-400", "representation"); + late.setId(20L); + RagIndexOperation operation = new RagIndexOperation( + project, "develop", "develop-400", "develop-401", "late-key"); + operation.setId(30L); + operation.setGeneration(late); + operation.start(); + when(operationRepository.findById(30L)).thenReturn(Optional.of(operation)); + when(branchIndexRepository.findByIdForPublication(10L)) + .thenReturn(Optional.of(branchIndex)); + + service.fail(30L, "late worker failed"); + + assertThat(late.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.FAILED); + assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.FAILED); + assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-402"); + assertThat(branchIndex.getLifecycleStatus()) + .isEqualTo(RagBranchIndexLifecycleStatus.BUILDING); + assertThat(branchIndex.getErrorMessage()).isNull(); + verify(branchIndexRepository, never()).save(branchIndex); + } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java index 8956f29db..417fc0336 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java @@ -857,13 +857,26 @@ void updateBranchIndexRejectsBranchOutsideRetainedConfiguration() { } @Test - void testUpdateBranchIndex_EmptyDiff() throws Exception { + void updateBranchIndexEmptyDiffSeedsExactSnapshot() throws Exception { + RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); + BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); + service = new RagOperationsServiceImpl( + ragIndexTrackingService, incrementalRagUpdateService, + analysisLockService, analysisJobService, + ragBranchIndexRepository, vcsClientProvider, + ragPipelineClient, registry, builder); setupRagEnabled(); setupVcsBinding(); + service = spy(service); + doReturn(true).when(service).shouldHaveBranchIndex(testProject, "feature"); when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); VcsClient mockVcs = mock(VcsClient.class); when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); when(mockVcs.getBranchDiff("my-workspace", "my-repo", "main", "feature")).thenReturn(""); + when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) + .thenReturn("feature-head"); + doReturn(true).when(service).triggerIncrementalUpdate( + eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); @SuppressWarnings("unchecked") Consumer> eventConsumer = mock(Consumer.class); @@ -871,6 +884,8 @@ void testUpdateBranchIndex_EmptyDiff() throws Exception { assertThat(result).isTrue(); verify(eventConsumer).accept(argThat(m -> "info".equals(m.get("type")))); + verify(service).triggerIncrementalUpdate( + testProject, "feature", "feature-head", "", eventConsumer); } @Test diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java index 7119dcfc0..89af6e8c3 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/service/PipelineJobService.java @@ -126,6 +126,26 @@ public Job createRagIndexJob(Project project, boolean isInitial, JobTriggerSourc ); } + @Override + public Job createRagIndexJob( + Project project, + boolean isInitial, + JobTriggerSource triggerSource, + String branchName, + String commitHash) { + log.info("Creating branch-bound RAG {} job for project: {}, branch: {} (trigger: {})", + isInitial ? "initial indexing" : "incremental update", + project.getName(), branchName, triggerSource); + return jobService.createRagIndexJob( + project, + isInitial, + triggerSource, + null, + branchName, + commitHash + ); + } + /** * Alias for createRagIndexJob for backward compatibility. * Used by pipeline processors for RAG jobs triggered by webhooks. diff --git a/python-ecosystem/rag-pipeline/Dockerfile b/python-ecosystem/rag-pipeline/Dockerfile index 0a35959d9..3bbb380d0 100644 --- a/python-ecosystem/rag-pipeline/Dockerfile +++ b/python-ecosystem/rag-pipeline/Dockerfile @@ -51,7 +51,7 @@ ENV TRANSFORMERS_CACHE=/tmp/.transformers_cache ENV HF_HOME=/tmp/.huggingface ENV LLAMA_INDEX_CACHE_DIR=/tmp/.llama_index # Allow concurrent indexing by running multiple Uvicorn workers -ENV UVICORN_WORKERS=4 +ENV UVICORN_WORKERS=1 # CPU Threading Optimization (can be overridden via .env) # These control parallelism for numerical operations diff --git a/python-ecosystem/rag-pipeline/Dockerfile.observable b/python-ecosystem/rag-pipeline/Dockerfile.observable index b828eb65f..c2e302046 100644 --- a/python-ecosystem/rag-pipeline/Dockerfile.observable +++ b/python-ecosystem/rag-pipeline/Dockerfile.observable @@ -51,7 +51,7 @@ ENV TRANSFORMERS_CACHE=/tmp/.transformers_cache ENV HF_HOME=/tmp/.huggingface ENV LLAMA_INDEX_CACHE_DIR=/tmp/.llama_index # Allow concurrent indexing by running multiple Uvicorn workers -ENV UVICORN_WORKERS=4 +ENV UVICORN_WORKERS=1 # CPU Threading Optimization (can be overridden via .env) # These control parallelism for numerical operations diff --git a/python-ecosystem/rag-pipeline/main.py b/python-ecosystem/rag-pipeline/main.py index 38226ecdb..6e3df0f8b 100644 --- a/python-ecosystem/rag-pipeline/main.py +++ b/python-ecosystem/rag-pipeline/main.py @@ -125,10 +125,12 @@ def validate_environment(): from rag_pipeline.api.api import app if __name__ == "__main__": - # Use multiple workers to allow concurrent indexing requests - # Each worker can handle one long-running indexing task - workers = int(os.environ.get("UVICORN_WORKERS", "4")) - logger.info(f"Starting Uvicorn with {workers} workers for concurrent request handling") + # Keep one process by default. Each worker initializes its own embedding and + # indexing state, so multiplying Uvicorn workers multiplies the memory cost + # of concurrent branch snapshots. Branch-level parallelism is bounded by + # the Java maintenance executor instead. + workers = int(os.environ.get("UVICORN_WORKERS", "1")) + logger.info(f"Starting Uvicorn with {workers} worker process(es)") uvicorn.run( "rag_pipeline.api.api:app", host="0.0.0.0", diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py index 918a612cc..aeb8f473e 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/index.py @@ -1,7 +1,8 @@ """Index and branch management endpoints.""" +import asyncio import json import logging -from queue import Queue +from queue import Empty, Queue from threading import Thread from typing import List from fastapi import APIRouter, HTTPException, BackgroundTasks, Query @@ -139,7 +140,7 @@ def index_repository_stream(request: IndexRequest): """ _, index_manager = _get_singletons() - def event_stream(): + async def event_stream(): events: Queue[tuple[str, object]] = Queue() def progress(event: dict) -> None: @@ -177,9 +178,21 @@ def run_index() -> None: logger.error("Error indexing repository with progress: %s", exception) events.put(("error", {"message": str(exception)})) - Thread(target=run_index, name="rag-index-progress", daemon=True).start() + worker = Thread( + target=run_index, + name="rag-index-progress", + daemon=True, + ) + worker.start() while True: - event_type, payload = events.get() + try: + event_type, payload = events.get_nowait() + except Empty: + # Polling a thread-safe queue avoids nesting a blocking queue + # consumer inside Starlette's thread pool. The short wait keeps + # the event loop responsive and progress delivery prompt. + await asyncio.sleep(0.05) + continue if event_type == "progress": event = {"type": "progress", **payload} elif event_type == "complete": @@ -188,6 +201,10 @@ def run_index() -> None: event = {"type": "error", **payload} yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" if event_type in {"complete", "error"}: + # The terminal event is scheduled just before the producer + # returns. Join that final unwind so a short-lived consumer + # cannot finish while its producer thread is still active. + worker.join(timeout=1.0) break return StreamingResponse(event_stream(), media_type="text/event-stream") @@ -318,11 +335,10 @@ def advance_generation(request: AdvanceGenerationRequest): @router.post("/index/generation-aliases") def publish_generation_aliases(request: GenerationAliasPublicationRequest): - """Repair readable aliases for a completed immutable generation. + """Publish or repair readable aliases for an accepted immutable generation. - This is deliberately separate from indexing. Indexing publishes all aliases - atomically; this endpoint makes deployments backward-compatible by repairing - active generations that predate readable branch aliases. + This is deliberately separate from indexing so the Java registry can reject + a stale completed build before any mutable branch-head alias is moved. """ _, index_manager = _get_singletons() try: diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py index 39bd10905..016fc87a1 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py @@ -130,6 +130,14 @@ def index_pr_files(request: PRIndexRequest): requested_collection_target = None if not isinstance(requested_base_manifest, str): requested_base_manifest = None + if requested_collection_target and not request.base_revision: + raise IncrementalIndexPreconditionError( + "PR overlay collection target requires an exact base revision" + ) + if requested_base_manifest and not request.base_revision: + raise IncrementalIndexPreconditionError( + "PR overlay base generation receipt requires an exact base revision" + ) exact_binding = bool( request.base_revision and (requested_collection_target or requested_base_manifest) @@ -836,7 +844,10 @@ def delete_pr_files( collection_name=collection_name, points_selector=Filter( must=[ - FieldCondition(key="pr_number", match=MatchValue(value=pr_number)) + FieldCondition(key="workspace", match=MatchValue(value=workspace)), + FieldCondition(key="project", match=MatchValue(value=project)), + FieldCondition(key="pr", match=MatchValue(value=True)), + FieldCondition(key="pr_number", match=MatchValue(value=pr_number)), ] ) ) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py index 826e38cad..52eaef43a 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/query.py @@ -42,6 +42,38 @@ def _authoritative_pr_branch(request: PRContextRequest) -> Optional[str]: return request.branch +def _require_complete_pr_overlay_binding( + *, + pr_number: Optional[int], + target_branch: Optional[str], + source_revision: Optional[str], + base_revision: Optional[str], + base_generation_manifest: Optional[str], + pr_generation_fingerprint: Optional[str], + pr_overlay_manifest: Optional[str], +) -> bool: + """Reject a claimed exact overlay whose identity is incomplete.""" + overlay_binding_requested = bool( + pr_generation_fingerprint or pr_overlay_manifest + ) + if not overlay_binding_requested: + return False + if not all(( + pr_number, + target_branch, + source_revision, + base_revision, + base_generation_manifest, + pr_generation_fingerprint, + pr_overlay_manifest, + )): + raise IncrementalIndexPreconditionError( + "revision-bound PR overlay requires PR number, one authoritative " + "branch, source/base revisions, and both generation receipts" + ) + return True + + @router.post("/query/search") def semantic_search(request: QueryRequest): """Perform semantic search.""" @@ -134,6 +166,15 @@ def get_pr_context(request: PRContextRequest): request.pr_overlay_generation_manifest_sha256 ) collection_target = _optional_string(request.collection_target) + exact_overlay_binding = _require_complete_pr_overlay_binding( + pr_number=request.pr_number, + target_branch=authoritative_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest=base_generation_manifest, + pr_generation_fingerprint=pr_generation_fingerprint, + pr_overlay_manifest=pr_overlay_manifest, + ) receipt = None overlay_receipt = None if base_revision: @@ -165,13 +206,7 @@ def get_pr_context(request: PRContextRequest): # HYBRID MODE: Query PR-indexed data first if pr_number is provided if request.pr_number: - if all(( - source_revision, - base_revision, - base_generation_manifest, - pr_generation_fingerprint, - pr_overlay_manifest, - )): + if exact_overlay_binding: overlay_receipt = read_pr_overlay_generation( index_manager.qdrant_client, collection_name, @@ -564,6 +599,15 @@ def get_deterministic_context(request: DeterministicContextRequest): collection_target = _optional_string(request.collection_target) receipt = None overlay_receipt = None + exact_overlay_binding = _require_complete_pr_overlay_binding( + pr_number=request.pr_number, + target_branch=target_branch, + source_revision=source_revision, + base_revision=base_revision, + base_generation_manifest=base_generation_manifest, + pr_generation_fingerprint=pr_generation_fingerprint, + pr_overlay_manifest=pr_overlay_manifest, + ) if base_revision and target_branch: if len(request.branches) != 1: raise IncrementalIndexPreconditionError( @@ -578,14 +622,6 @@ def get_deterministic_context(request: DeterministicContextRequest): generation_manifest_sha256=base_generation_manifest, collection_target=collection_target, ) - exact_overlay_binding = all(( - request.pr_number, - source_revision, - base_revision, - base_generation_manifest, - pr_generation_fingerprint, - pr_overlay_manifest, - )) if exact_overlay_binding: overlay_receipt = read_pr_overlay_generation( index_manager.qdrant_client, diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py index c769abeba..c990a869f 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py @@ -38,6 +38,10 @@ ) from .. import revision_preflight from ..repository_overlay import IncrementalIndexPreconditionError +from ..revision_preflight_cache import ( + RevisionPreflightCache, + RevisionPreflightKey, +) from ..source_tree import ( compute_repository_source_tree_sha256, verify_repository_source_tree, @@ -69,6 +73,16 @@ def _config_int(config, name: str, default: int) -> int: return default +def _config_nonnegative_int(config, name: str, default: int) -> int: + value = getattr(config, name, default) + if isinstance(value, bool) or not isinstance(value, (int, str)): + return default + try: + return max(0, int(value)) + except (TypeError, ValueError): + return default + + def _config_float(config, name: str, default: float) -> float: value = getattr(config, name, default) if not isinstance(value, (int, float, str)): @@ -100,6 +114,23 @@ def __init__(self, config: RAGConfig): 5.0, ), ) + self._revision_preflight_cache = RevisionPreflightCache( + max_entries=_config_int( + config, + "revision_preflight_cache_entries", + 512, + ), + ttl_seconds=_config_nonnegative_int( + config, + "revision_preflight_cache_ttl_seconds", + 0, + ), + max_concurrent_loads=_config_int( + config, + "revision_preflight_max_concurrency", + 2, + ), + ) self.index_representation_fingerprint = ( index_representation_fingerprint(config) ) @@ -296,6 +327,12 @@ def index_repository( progress_callback: Optional[Callable[[dict], None]] = None, ) -> IndexStats: """Index entire repository for a branch using atomic swap strategy.""" + if collection_target is None and ( + publish_branch_alias or publish_legacy_project_alias + ): + raise ValueError( + "readable generation aliases require an immutable collection target" + ) alias_name = collection_target or self._get_project_collection_name( workspace, project ) @@ -357,22 +394,56 @@ def get_revision_preflight( physical = self._collection_manager.resolve_collection_target(target) if physical is None: return None - result = read_repository_revision_preflight( - self.qdrant_client, - physical, - branch, - commit, - ) - if result is None: - return None - if ( - result.get("workspace") != workspace - or result.get("project") != project - ): - raise IncrementalIndexPreconditionError( - "repository generation coordinates do not match the requested tenant" + cache = getattr(self, "_revision_preflight_cache", None) + if cache is None: + # Preserve lightweight construction used by tooling and tests. + cache = RevisionPreflightCache( + max_entries=_config_int( + self.config, + "revision_preflight_cache_entries", + 512, + ), + ttl_seconds=_config_nonnegative_int( + self.config, + "revision_preflight_cache_ttl_seconds", + 0, + ), + max_concurrent_loads=_config_int( + self.config, + "revision_preflight_max_concurrency", + 2, + ), + ) + self._revision_preflight_cache = cache + + def verify(): + result = read_repository_revision_preflight( + self.qdrant_client, + physical, + branch, + commit, ) - return result + if result is None: + return None + if ( + result.get("workspace") != workspace + or result.get("project") != project + ): + raise IncrementalIndexPreconditionError( + "repository generation coordinates do not match the requested tenant" + ) + return result + + return cache.get_or_load( + RevisionPreflightKey( + collection=physical, + workspace=workspace, + project=project, + branch=branch, + commit=commit, + ), + verify, + ) def publish_generation_aliases( self, @@ -384,11 +455,10 @@ def publish_generation_aliases( publish_branch_alias: bool = True, publish_legacy_project_alias: bool = False, ) -> List[str]: - """Repair operator aliases after proving one sealed generation identity. + """Publish operator aliases after proving one sealed generation identity. - Normal indexing publishes these pointers atomically with its immutable - target alias. This idempotent path exists only for generations created - before that contract was introduced or when Qdrant has been restored. + The Java registry calls this only after accepting the generation as the + active branch head. It is also an idempotent repair path after restore. """ aliases = self._publication_aliases( workspace, @@ -635,11 +705,12 @@ def advance_generation( raise IncrementalIndexPreconditionError( "source repository generation is unavailable" ) - source_receipt = read_repository_revision_preflight( - self.qdrant_client, - source_physical, + source_receipt = self.get_revision_preflight( + workspace, + project, branch, source_commit, + collection_target=source_physical, ) if source_receipt is None: raise IncrementalIndexPreconditionError( @@ -650,11 +721,12 @@ def advance_generation( target_collection_target ) if target_physical is not None: - existing = read_repository_revision_preflight( - self.qdrant_client, - target_physical, + existing = self.get_revision_preflight( + workspace, + project, branch, commit, + collection_target=target_physical, ) if existing is None: raise IncrementalIndexPreconditionError( diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py index bda343f51..96825c994 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight.py @@ -4,6 +4,7 @@ import hashlib import json +from types import SimpleNamespace from qdrant_client.models import FieldCondition, Filter, MatchValue @@ -371,7 +372,8 @@ def read_repository_revision_preflight( point_count = 0 identity = None manifest_points = [] - member_points = [] + members = [] + member_validation_error = None offset = None while True: @@ -404,9 +406,26 @@ def read_repository_revision_preflight( "fully reindex the revision" ) if payload.get(GENERATION_MANIFEST_PAYLOAD_KEY) is True: - manifest_points.append(point) + # Retain only the small manifest payload. Keeping its 4096-d + # vector would otherwise pin one full Qdrant point until the + # complete generation scan finishes. + manifest_points.append(SimpleNamespace(payload=dict(payload))) else: - member_points.append(point) + try: + # Verify while this page is live and retain only the + # compact (point id, digest) receipt. Never accumulate + # complete payloads and vectors for the whole repository. + members.append(verified_generation_member(point)) + except GenerationManifestError as exception: + # Preserve validation priority from the original preflight: + # mixed identity/revision and missing-manifest diagnostics + # are established before member-content failure. Keep only + # the error text so its traceback cannot pin this page. + if member_validation_error is None: + member_validation_error = str(exception) + point = None + payload = None + del points if offset is None: break @@ -426,16 +445,11 @@ def read_repository_revision_preflight( branch, commit, ) - try: - members = [ - verified_generation_member(point) - for point in member_points - ] - except GenerationManifestError as exception: + if member_validation_error is not None: raise IncrementalIndexPreconditionError( "exact revision repository generation member content failed " "integrity validation; fully reindex the revision" - ) from exception + ) generation_identity = _validate_generation_manifest( manifest_points, members, diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight_cache.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight_cache.py new file mode 100644 index 000000000..cf8b72b06 --- /dev/null +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/revision_preflight_cache.py @@ -0,0 +1,144 @@ +"""Bounded, single-flight cache for immutable revision verification receipts.""" + +from __future__ import annotations + +import copy +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass +from typing import Callable, Hashable, Optional + + +@dataclass(frozen=True) +class RevisionPreflightKey: + """Identity of one tenant-bound immutable physical generation.""" + + collection: str + workspace: str + project: str + branch: str + commit: str + + +@dataclass +class _CacheEntry: + expires_at: float | None + value: Optional[dict] + + +@dataclass +class _Flight: + event: threading.Event + value: Optional[dict] = None + error: BaseException | None = None + + +class RevisionPreflightCache: + """Cache expensive immutable-generation verification without stampedes. + + Only the compact verification receipt is retained. A bounded semaphore + limits cold verification across different generations, while callers for + the same generation share one in-flight load. Loader failures are shared + with current waiters but are never cached, so a transient Qdrant failure + remains retryable. + """ + + def __init__( + self, + *, + max_entries: int, + ttl_seconds: float, + max_concurrent_loads: int, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if max_entries < 1: + raise ValueError("max_entries must be positive") + if ttl_seconds < 0: + raise ValueError("ttl_seconds must not be negative") + if max_concurrent_loads < 1: + raise ValueError("max_concurrent_loads must be positive") + self._max_entries = max_entries + self._ttl_seconds = ttl_seconds + self._clock = clock + self._lock = threading.Lock() + self._values: OrderedDict[Hashable, _CacheEntry] = OrderedDict() + self._flights: dict[Hashable, _Flight] = {} + self._load_slots = threading.BoundedSemaphore(max_concurrent_loads) + + def get_or_load( + self, + key: Hashable, + loader: Callable[[], Optional[dict]], + ) -> Optional[dict]: + """Return a cached receipt or execute one bounded single-flight load.""" + now = self._clock() + with self._lock: + entry = self._values.get(key) + if entry is not None and ( + entry.expires_at is None or entry.expires_at > now + ): + self._values.move_to_end(key) + return copy.deepcopy(entry.value) + if entry is not None: + self._values.pop(key, None) + + flight = self._flights.get(key) + owner = flight is None + if owner: + flight = _Flight(event=threading.Event()) + self._flights[key] = flight + + if not owner: + flight.event.wait() + if flight.error is not None: + raise flight.error + return copy.deepcopy(flight.value) + + try: + with self._load_slots: + value = loader() + except BaseException as exception: + with self._lock: + self._flights.pop(key, None) + flight.error = exception + flight.event.set() + raise + + stored_value = copy.deepcopy(value) + with self._lock: + # An absent revision can later appear in a legacy mutable target. + # Positive receipts belong to sealed immutable generations; cache + # those, but keep absence immediately observable and retryable. + if stored_value is not None: + self._values[key] = _CacheEntry( + expires_at=( + None + if self._ttl_seconds == 0 + else self._clock() + self._ttl_seconds + ), + value=stored_value, + ) + self._values.move_to_end(key) + while len(self._values) > self._max_entries: + self._values.popitem(last=False) + self._flights.pop(key, None) + flight.value = stored_value + flight.event.set() + return copy.deepcopy(stored_value) + + def invalidate_collection(self, collection: str) -> None: + """Discard cached receipts for one physical collection.""" + with self._lock: + keys = [ + key + for key in self._values + if getattr(key, "collection", None) == collection + ] + for key in keys: + self._values.pop(key, None) + + def clear(self) -> None: + """Discard all completed cache entries without disturbing loaders.""" + with self._lock: + self._values.clear() diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py index 7a81a2519..163b39aa9 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/models/config.py @@ -113,6 +113,24 @@ class RAGConfig(BaseModel): ), ge=0, ) + revision_preflight_cache_entries: int = Field( + default_factory=lambda: int( + os.getenv("RAG_REVISION_PREFLIGHT_CACHE_ENTRIES", "512") + ), + ge=1, + ) + revision_preflight_cache_ttl_seconds: int = Field( + default_factory=lambda: int( + os.getenv("RAG_REVISION_PREFLIGHT_CACHE_TTL_SECONDS", "0") + ), + ge=0, + ) + revision_preflight_max_concurrency: int = Field( + default_factory=lambda: int( + os.getenv("RAG_REVISION_PREFLIGHT_MAX_CONCURRENCY", "2") + ), + ge=1, + ) # Embedding dimensions - auto-detected from model or set via env var embedding_dim: int = Field(default_factory=lambda: int(os.getenv("EMBEDDING_DIM", "0"))) diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py index 28ecaa0c2..65b335ff7 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/services/deterministic_context.py @@ -279,6 +279,24 @@ def get_deterministic_context( Returns: Dict with chunks grouped by retrieval type and rich metadata """ + if pr_generation_fingerprint and not all(( + pr_number, + pr_source_revision, + pr_base_revision, + pr_base_generation_manifest_sha256, + )): + raise IncrementalIndexPreconditionError( + "PR generation fingerprint requires PR number and complete " + "source/base generation identity" + ) + if pr_generation_fingerprint and len([ + branch for branch in branches if branch + ]) != 1: + raise IncrementalIndexPreconditionError( + "revision-bound deterministic context requires exactly one " + "authoritative branch" + ) + collection_name = ( collection_target or self._get_project_collection_name(workspace, project) diff --git a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py index d5ad8f361..f7c22c9ef 100644 --- a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py +++ b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py @@ -205,24 +205,50 @@ def test_advance_is_idempotent_when_exact_target_already_exists(): manager._stats_manager = MagicMock() expected = SimpleNamespace(generation_manifest_sha256="e" * 64) manager._stats_manager.get_branch_stats.return_value = expected + manager.get_revision_preflight = MagicMock(return_value={ + "generation_manifest_sha256": "e" * 64, + "source_tree_sha256": SOURCE_TREE, + }) - import rag_pipeline.core.index_manager.manager as manager_module - original = manager_module.read_repository_revision_preflight - manager_module.read_repository_revision_preflight = ( - lambda *_args, **_kwargs: {"generation_manifest_sha256": "e" * 64} + result = manager.advance_generation( + "source", "target", SOURCE_COMMIT, SOURCE_TREE, [], [], None, + "ws", "project", "develop", TARGET_COMMIT, ) - try: - result = manager.advance_generation( - "source", "target", SOURCE_COMMIT, SOURCE_TREE, [], [], None, - "ws", "project", "develop", TARGET_COMMIT, - ) - finally: - manager_module.read_repository_revision_preflight = original assert result is expected manager._collection_manager.create_pending_collection.assert_not_called() +def test_existing_target_is_not_published_when_tenant_preflight_rejects_it(): + manager = object.__new__(RAGIndexManager) + manager._mutation_coordinator = _Coordinator() + manager._collection_manager = MagicMock() + manager._collection_manager.resolve_collection_target.side_effect = [ + "source-physical", "foreign-target-physical" + ] + manager.qdrant_client = MagicMock() + manager._stats_manager = MagicMock() + manager.config = SimpleNamespace(qdrant_collection_prefix="rag") + manager.get_revision_preflight = MagicMock(side_effect=[ + {"generation_manifest_sha256": "1" * 64}, + IncrementalIndexPreconditionError( + "repository generation coordinates do not match the requested tenant" + ), + ]) + + with pytest.raises( + IncrementalIndexPreconditionError, + match="coordinates do not match", + ): + manager.advance_generation( + "source", "foreign-target", SOURCE_COMMIT, SOURCE_TREE, + [], [], None, "ws", "project", "develop", TARGET_COMMIT, + publish_branch_alias=True, + ) + + manager._collection_manager.atomic_assign_aliases.assert_not_called() + + def test_exact_generation_delete_verifies_tenant_coordinates(): client = QdrantClient(":memory:") client.create_collection( diff --git a/python-ecosystem/rag-pipeline/tests/test_index_manager.py b/python-ecosystem/rag-pipeline/tests/test_index_manager.py index fa1808cb2..d8ad3b9c1 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_manager.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_manager.py @@ -428,6 +428,24 @@ def test_branch_operator_alias_is_readable_and_branch_specific(self): "Workspace", "Project", "release/1.2" ).startswith("rag_workspace__project__release_1_2_") + def test_readable_alias_publication_requires_immutable_generation_target(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + + with pytest.raises( + ValueError, + match="require an immutable collection target", + ): + manager.index_repository( + repo_path="/tmp/repository", + workspace="workspace", + project="project", + branch="main", + commit="abc123", + publish_branch_alias=True, + ) + @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") @patch("rag_pipeline.core.index_manager.manager.QdrantClient") diff --git a/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py b/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py index 3bd6eb62b..60f0c65f9 100644 --- a/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py +++ b/python-ecosystem/rag-pipeline/tests/test_revision_bound_queries.py @@ -21,6 +21,7 @@ from rag_pipeline.services.base import RAGQueryBase from rag_pipeline.services.pr_context import PRContextMixin from rag_pipeline.services.semantic_search import SemanticSearchMixin +from rag_pipeline.services.deterministic_context import DeterministicContextMixin BASE_REVISION = "a" * 40 @@ -526,3 +527,71 @@ def test_revision_bound_deterministic_context_rejects_extra_branch(): assert exception.value.status_code == 409 assert "exactly one authoritative" in exception.value.detail service.get_deterministic_context.assert_not_called() + + +def test_deterministic_context_rejects_partial_overlay_identity(): + manager = _manager() + service = MagicMock() + request = DeterministicContextRequest( + workspace="ws", + project="project", + branches=["main"], + file_paths=["src/Foo.php"], + pr_number=42, + pr_generation_fingerprint=PR_GENERATION, + ) + + with patch( + "rag_pipeline.api.routers.query._get_singletons", + return_value=(manager, service), + ): + with pytest.raises(HTTPException) as exception: + get_deterministic_context(request) + + assert exception.value.status_code == 409 + assert "requires PR number" in exception.value.detail + service.get_deterministic_context.assert_not_called() + + +def test_exact_overlay_rejects_empty_authoritative_branch_list(): + manager = _manager() + service = MagicMock() + request = DeterministicContextRequest( + workspace="ws", + project="project", + branches=[], + file_paths=["src/Foo.php"], + pr_number=42, + source_revision=SOURCE_REVISION, + base_revision=BASE_REVISION, + base_generation_manifest_sha256=BASE_GENERATION, + pr_generation_fingerprint=PR_GENERATION, + pr_overlay_generation_manifest_sha256=PR_OVERLAY_MANIFEST, + ) + + with patch( + "rag_pipeline.api.routers.query._get_singletons", + return_value=(manager, service), + ): + with pytest.raises(HTTPException) as exception: + get_deterministic_context(request) + + assert exception.value.status_code == 409 + assert "one authoritative branch" in exception.value.detail + service.get_deterministic_context.assert_not_called() + + +def test_deterministic_service_rejects_fingerprint_without_revisions(): + with pytest.raises( + IncrementalIndexPreconditionError, + match="complete source/base generation identity", + ): + DeterministicContextMixin.get_deterministic_context( + MagicMock(), + workspace="ws", + project="project", + branches=["main"], + file_paths=["src/Foo.php"], + pr_number=42, + pr_generation_fingerprint=PR_GENERATION, + ) diff --git a/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py b/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py index 51a4bfda9..2283e1fd1 100644 --- a/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py +++ b/python-ecosystem/rag-pipeline/tests/test_revision_preflight.py @@ -484,6 +484,39 @@ def test_exact_revision_preflight_rejects_legacy_unsealed_revision(): ) +def test_exact_revision_preflight_verifies_members_while_scrolling_pages(): + payload = _identity_payload(path="Example.php", text=" Date: Mon, 10 Aug 2026 02:04:08 +0300 Subject: [PATCH 3/6] Bugfixes & Improvements fix #1: improve PR review reporting, RAG consistency, and concurrent analysis fix #2: prevent out-of-diff findings from rejecting PR reviews fix #3: handle empty Jira task descriptions ( qa-doc ) --- .../config/inference-orchestrator/.env.sample | 2 +- .../java-shared/application.properties.sample | 5 + .../analysis/BranchAnalysisProcessor.java | 30 ++- .../branch/BranchAnalysisGateService.java | 80 ++++++++ .../branch/BranchAnalysisGateServiceTest.java | 22 +++ .../repository/job/JobRepository.java | 20 ++ .../actions/CommentOnPullRequestAction.java | 42 ++++- .../actions/GetPullRequestDiffAction.java | 84 +++++++-- .../CommentOnPullRequestActionTest.java | 75 ++++++++ .../actions/GetPullRequestDiffActionTest.java | 39 ++++ .../BitbucketCloudBranchWebhookHandler.java | 3 +- .../pipelineagent/config/AsyncConfig.java | 11 +- .../ProviderPipelineActionController.java | 6 +- .../processor/PipelineActionProcessor.java | 33 +++- .../processor/WebhookAsyncProcessor.java | 14 +- .../service/GitHubReportingService.java | 104 +++++++++-- .../github/service/GitHubReviewFormatter.java | 171 +++++++++++++++++- .../GitHubBranchWebhookHandler.java | 3 +- .../GitHubPullRequestWebhookHandler.java | 3 +- .../GitLabBranchWebhookHandler.java | 3 +- .../GitLabMrMergeWebhookHandler.java | 3 +- .../pipelineagent/config/AsyncConfigTest.java | 40 ++++ ...lineActionProcessorDependencyGateTest.java | 103 +++++++++++ .../WebhookAsyncProcessorBranchGateTest.java | 39 +++- .../service/GitHubReportingServiceTest.java | 104 ++++++++++- .../service/GitHubReviewFormatterTest.java | 87 ++++++++- .../integration/test_qa_documentation.py | 24 +++ .../src/api/routers/qa_documentation.py | 13 +- .../src/server/queue_consumer.py | 2 +- .../src/service/review/review_service.py | 2 +- .../rag-pipeline/integration/conftest.py | 1 + .../src/rag_pipeline/api/routers/pr.py | 6 +- .../core/index_manager/manager.py | 20 ++ .../rag-pipeline/tests/test_coordination.py | 37 ++++ .../rag-pipeline/tests/test_router_pr.py | 18 +- 35 files changed, 1145 insertions(+), 104 deletions(-) create mode 100644 java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java create mode 100644 java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessorDependencyGateTest.java diff --git a/deployment/config/inference-orchestrator/.env.sample b/deployment/config/inference-orchestrator/.env.sample index 6f4524f59..95f6ef622 100644 --- a/deployment/config/inference-orchestrator/.env.sample +++ b/deployment/config/inference-orchestrator/.env.sample @@ -11,7 +11,7 @@ SERVICE_SECRET=change-me-to-a-random-secret # AI_CLIENT_HOST=0.0.0.0 # AI_CLIENT_PORT=8000 # REDIS_URL=redis://localhost:6379/1 -# MAX_CONCURRENT_REVIEWS=4 +# MAX_CONCURRENT_REVIEWS=20 # ANALYSIS_QUEUE_HEARTBEAT_SECONDS=30 # ANALYSIS_CONSUMER_HEARTBEAT_SECONDS=5 # MAX_CONCURRENT_COMMANDS=10 diff --git a/deployment/config/java-shared/application.properties.sample b/deployment/config/java-shared/application.properties.sample index 837fa4a63..c788fb31a 100644 --- a/deployment/config/java-shared/application.properties.sample +++ b/deployment/config/java-shared/application.properties.sample @@ -213,6 +213,11 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # Lock cleanup interval - how often to clean up expired locks (in milliseconds) #analysis.lock.cleanup.interval.ms=300000 +# Durable webhook execution. The executor has no in-memory queue: accepted work +# beyond max capacity stays QUEUED in PostgreSQL and is recovered later. +#webhook.executor.core-pool-size=8 +#webhook.executor.max-pool-size=20 + # Hard PR-wide spending limits are supplied to pipeline-agent through deployment/.env: # ANALYSIS_MAX_FILES=150 # ANALYSIS_MAX_FILE_SIZE_BYTES=5242880 diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java index 21d34487b..ea3e8f93f 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java @@ -153,13 +153,33 @@ public BranchAnalysisProcessor( public Map process( BranchProcessRequest request, Consumer> consumer) throws IOException { + return process(request, consumer, false); + } + + /** + * Execute a branch job whose persisted cross-analysis dependencies were + * already resolved by the dispatcher. + */ + public Map processAfterDependencyGate( + BranchProcessRequest request, + Consumer> consumer) throws IOException { + return process(request, consumer, true); + } + + private Map process( + BranchProcessRequest request, + Consumer> consumer, + boolean dependencyGateSatisfied) throws IOException { Project project = projectService.getProjectWithConnections(request.getProjectId()); - // PR jobs are registered before async processing starts and remain active - // until their source-branch lock is released and analysis is persisted. - branchAnalysisGateService.awaitPrAnalysis( - project.getId(), request.getTargetBranchName(), - request.getSourcePrNumber(), consumer); + if (!dependencyGateSatisfied) { + // Scheduled/direct callers without a durable dispatch job retain the + // broad compatibility barrier. Webhook and pipeline dispatchers use the + // job-id snapshot barrier before calling processAfterDependencyGate(). + branchAnalysisGateService.awaitPrAnalysis( + project.getId(), request.getTargetBranchName(), + request.getSourcePrNumber(), consumer); + } refreshMergedBranchHead(project, request); Optional lockKey = analysisLockService.acquireLockWithWait( diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java index 2314f764a..6718f620e 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateService.java @@ -42,6 +42,36 @@ public BranchAnalysisGateService(JobRepository jobRepository) { this.jobRepository = jobRepository; } + /** + * Apply the dependency barrier for a durably accepted analysis job. + * Jobs only wait for older work on the same target branch, which preserves + * repository/RAG ordering without serializing independent PR reviews. + */ + public GateResult awaitDependencies( + Long projectId, + Job job, + Consumer> consumer) { + if (job == null || job.getId() == null) { + return GateResult.READY; + } + if (job.getJobType() == JobType.BRANCH_ANALYSIS) { + return awaitTurn( + projectId, + job.getBranchName(), + job.getId(), + job.getPrNumber(), + consumer); + } + if (job.getJobType() == JobType.PR_ANALYSIS) { + awaitOlderBranchAnalyses( + projectId, + job.getBranchName(), + job.getId(), + consumer); + } + return GateResult.READY; + } + /** * Wait for the relevant target-branch PR job. When the merge PR number is * known, only its newest analysis attempt can block reconciliation. When it @@ -117,6 +147,36 @@ public void awaitPrAnalysis( awaitTurn(projectId, branchName, null, sourcePrNumber, consumer); } + public void awaitOlderBranchAnalyses( + Long projectId, + String branchName, + Long currentPrJobId, + Consumer> consumer) { + if (branchName == null || branchName.isBlank() || currentPrJobId == null) { + return; + } + + long timeoutNanos = TimeUnit.MINUTES.toNanos(Math.max(1, waitTimeoutMinutes)); + long startedAt = System.nanoTime(); + while (jobRepository.existsActiveBranchAnalysisJobBefore( + projectId, branchName, currentPrJobId)) { + long waitedNanos = System.nanoTime() - startedAt; + if (waitedNanos >= timeoutNanos) { + log.warn("Timed out waiting for target branch update: project={}, branch={}, prJob={}, waited={}m", + projectId, branchName, currentPrJobId, + TimeUnit.NANOSECONDS.toMinutes(waitedNanos)); + throw new AnalysisLockedException( + AnalysisLockType.BRANCH_ANALYSIS.name(), branchName, projectId); + } + + emitBranchWait(consumer, branchName, waitedNanos); + if (!pause()) { + throw new AnalysisLockedException( + AnalysisLockType.BRANCH_ANALYSIS.name(), branchName, projectId); + } + } + } + private boolean hasBlockingPrAnalysis( Long projectId, String branchName, @@ -166,6 +226,26 @@ private void emitWait( } } + private void emitBranchWait( + Consumer> consumer, + String branchName, + long waitedNanos) { + if (consumer == null) { + return; + } + try { + consumer.accept(Map.of( + "type", "branch_analysis_wait", + "state", "waiting_for_target_branch", + "message", "Waiting for the earlier " + branchName + + " update and its RAG publication to finish", + "branchName", branchName, + "waitedSeconds", TimeUnit.NANOSECONDS.toSeconds(waitedNanos))); + } catch (Exception e) { + log.debug("Could not emit branch barrier status: {}", e.getMessage()); + } + } + private boolean pause() { if (pollIntervalMillis <= 0) { return true; diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java index 59cf8ba0a..98bd27097 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchAnalysisGateServiceTest.java @@ -147,6 +147,28 @@ void processorLevelBarrierUsesLatestAttemptForTheMergedPr() { verify(jobRepository, times(0)).existsActivePrAnalysisJob(1L, "main"); } + @Test + void prWaitsOnlyForOlderBranchJobsOnItsTargetBranch() { + @SuppressWarnings("unchecked") + Consumer> consumer = mock(Consumer.class); + Job prJob = job(JobStatus.RUNNING); + ReflectionTestUtils.setField(prJob, "id", 104L); + prJob.setBranchName("main"); + + when(jobRepository.existsActiveBranchAnalysisJobBefore(1L, "main", 104L)) + .thenReturn(true, true, false); + + BranchAnalysisGateService.GateResult result = service.awaitDependencies( + 1L, prJob, consumer); + + assertThat(result).isEqualTo(BranchAnalysisGateService.GateResult.READY); + verify(jobRepository, times(3)) + .existsActiveBranchAnalysisJobBefore(1L, "main", 104L); + verify(consumer, times(2)).accept(org.mockito.ArgumentMatchers.argThat( + event -> "branch_analysis_wait".equals(event.get("type")) + && "main".equals(event.get("branchName")))); + } + private static Job job(JobStatus status) { Job job = new Job(); job.setJobType(JobType.PR_ANALYSIS); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java index 8712ae2ea..28570720a 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java @@ -125,6 +125,26 @@ boolean existsActivePrAnalysisJobBefore( @Param("beforeJobId") Long beforeJobId ); + /** + * A PR accepted after a target-branch update must wait until that older + * branch job has reconciled repository state and published its RAG update. + * Later branch jobs wait for the PR instead, so the persisted job id gives + * both directions one deadlock-free ordering rule. + */ + @Query("SELECT CASE WHEN COUNT(j) > 0 THEN true ELSE false END FROM Job j " + + "WHERE j.project.id = :projectId AND j.branchName = :branchName " + + "AND j.jobType = org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS " + + "AND j.id < :beforeJobId " + + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + + "org.rostilos.codecrow.core.model.job.JobStatus.RUNNING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.WAITING)") + boolean existsActiveBranchAnalysisJobBefore( + @Param("projectId") Long projectId, + @Param("branchName") String branchName, + @Param("beforeJobId") Long beforeJobId + ); + /** * Return only the newest analysis attempt for a PR. An abandoned older * attempt must not poison branch reconciliation after a newer attempt has diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java index 33fb79db4..63210445f 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java @@ -332,13 +332,32 @@ public int deletePreviousReviewComments( String repo, int pullRequestNumber, String markerText + ) throws IOException { + return deletePreviousReviewComments( + owner, repo, pullRequestNumber, markerText, null); + } + + /** + * Delete earlier generated review comments while preserving one newly + * submitted replacement review. + */ + public int deletePreviousReviewComments( + String owner, + String repo, + int pullRequestNumber, + String markerText, + Long preservedReviewId ) throws IOException { int deleted = 0; for (Map comment : listReviewComments( owner, repo, pullRequestNumber)) { String body = (String) comment.get("body"); Number id = (Number) comment.get("id"); - if (body != null && body.contains(markerText) && id != null) { + Number reviewId = (Number) comment.get("pull_request_review_id"); + boolean preserved = preservedReviewId != null + && reviewId != null + && preservedReviewId.longValue() == reviewId.longValue(); + if (!preserved && body != null && body.contains(markerText) && id != null) { deleteReviewComment(owner, repo, id.longValue()); deleted++; } @@ -357,12 +376,31 @@ public int clearPreviousReviewBodies( int pullRequestNumber, String markerText, String clearedBody + ) throws IOException { + return clearPreviousReviewBodies( + owner, repo, pullRequestNumber, markerText, clearedBody, null); + } + + /** + * Clear earlier generated review summaries while preserving one newly + * submitted replacement review. + */ + public int clearPreviousReviewBodies( + String owner, + String repo, + int pullRequestNumber, + String markerText, + String clearedBody, + Long preservedReviewId ) throws IOException { int cleared = 0; for (Map review : listReviews(owner, repo, pullRequestNumber)) { String body = (String) review.get("body"); Number id = (Number) review.get("id"); - if (body != null && body.contains(markerText) && id != null) { + boolean preserved = preservedReviewId != null + && id != null + && preservedReviewId.longValue() == id.longValue(); + if (!preserved && body != null && body.contains(markerText) && id != null) { updateReviewBody( owner, repo, diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffAction.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffAction.java index cc5f1dee4..0750a4baf 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffAction.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffAction.java @@ -10,6 +10,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -57,6 +59,51 @@ public String getPullRequestDiff(String owner, String repo, int pullRequestNumbe */ private String getPullRequestDiffFromFiles(String owner, String repo, int pullRequestNumber) throws IOException { StringBuilder combinedDiff = new StringBuilder(); + for (PullRequestFilePatch file : getPullRequestFilePatches( + owner, repo, pullRequestNumber)) { + if (file.patch().isEmpty()) { + continue; + } + + // Build a unified diff header + String fromFile = "renamed".equals(file.status()) + && !file.previousFilename().isEmpty() + ? file.previousFilename() + : file.filename(); + combinedDiff.append("diff --git a/").append(fromFile) + .append(" b/").append(file.filename()).append("\n"); + + if ("added".equals(file.status())) { + combinedDiff.append("new file mode 100644\n"); + } else if ("removed".equals(file.status())) { + combinedDiff.append("deleted file mode 100644\n"); + } else if ("renamed".equals(file.status())) { + combinedDiff.append("rename from ").append(file.previousFilename()).append("\n"); + combinedDiff.append("rename to ").append(file.filename()).append("\n"); + } + + combinedDiff.append("--- a/").append(fromFile).append("\n"); + combinedDiff.append("+++ b/").append(file.filename()).append("\n"); + combinedDiff.append(file.patch()).append("\n"); + } + return combinedDiff.toString(); + } + + /** + * Return GitHub's structured per-file patches for a pull request. + * + *

The file endpoint is used directly for inline-review planning because + * its destination path is authoritative and does not require parsing quoted + * {@code diff --git} headers. A missing patch (for example, a binary or very + * large file) is preserved as an empty value so callers can treat its lines + * as unavailable for inline comments.

+ */ + public List getPullRequestFilePatches( + String owner, + String repo, + int pullRequestNumber + ) throws IOException { + List patches = new ArrayList<>(); String nextUrl = String.format("%s/repos/%s/%s/pulls/%d/files?per_page=100", GitHubConfig.API_BASE, owner, repo, pullRequestNumber); @@ -84,25 +131,8 @@ private String getPullRequestDiffFromFiles(String owner, String repo, int pullRe String patch = file.has("patch") ? file.get("patch").asText() : ""; String status = file.has("status") ? file.get("status").asText() : ""; String previousFilename = file.has("previous_filename") ? file.get("previous_filename").asText() : ""; - - if (!patch.isEmpty()) { - // Build a unified diff header - String fromFile = "renamed".equals(status) && !previousFilename.isEmpty() ? previousFilename : filename; - combinedDiff.append("diff --git a/").append(fromFile).append(" b/").append(filename).append("\n"); - - if ("added".equals(status)) { - combinedDiff.append("new file mode 100644\n"); - } else if ("removed".equals(status)) { - combinedDiff.append("deleted file mode 100644\n"); - } else if ("renamed".equals(status)) { - combinedDiff.append("rename from ").append(previousFilename).append("\n"); - combinedDiff.append("rename to ").append(filename).append("\n"); - } - - combinedDiff.append("--- a/").append(fromFile).append("\n"); - combinedDiff.append("+++ b/").append(filename).append("\n"); - combinedDiff.append(patch).append("\n"); - } + patches.add(new PullRequestFilePatch( + filename, previousFilename, status, patch)); } // Check for next page in Link header @@ -117,6 +147,20 @@ private String getPullRequestDiffFromFiles(String owner, String repo, int pullRe } } - return combinedDiff.toString(); + return List.copyOf(patches); + } + + public record PullRequestFilePatch( + String filename, + String previousFilename, + String status, + String patch + ) { + public PullRequestFilePatch { + filename = filename != null ? filename : ""; + previousFilename = previousFilename != null ? previousFilename : ""; + status = status != null ? status : ""; + patch = patch != null ? patch : ""; + } } } diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestActionTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestActionTest.java index c4cbac600..44d4f5595 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestActionTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestActionTest.java @@ -208,6 +208,42 @@ void deletePreviousReviewComments_deletesOnlyMarkedInlineComments() throws IOExc assertThat(requests.get(0).url().queryParameter("page")).isEqualTo("1"); } + @Test + void deletePreviousReviewComments_preservesTheReplacementReview() throws IOException { + List requests = new ArrayList<>(); + when(okHttpClient.newCall(any(Request.class))).thenAnswer(invocation -> { + Request request = invocation.getArgument(0); + requests.add(request); + + String responseJson = request.method().equals("GET") + ? "[{\"id\":41,\"pull_request_review_id\":51," + + "\"body\":\"old \"}," + + "{\"id\":42,\"pull_request_review_id\":99," + + "\"body\":\"new \"}]" + : "{}"; + Response requestResponse = new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create(responseJson, MediaType.parse("application/json"))) + .build(); + Call requestCall = mock(Call.class); + when(requestCall.execute()).thenReturn(requestResponse); + return requestCall; + }); + + int deleted = action.deletePreviousReviewComments( + "owner", "repo", 123, "", 99L); + + assertThat(deleted).isEqualTo(1); + assertThat(requests).extracting(request -> request.method() + " " + request.url().encodedPath()) + .containsExactly( + "GET /repos/owner/repo/pulls/123/comments", + "DELETE /repos/owner/repo/pulls/comments/41" + ); + } + @Test void clearPreviousReviewBodies_clearsOnlyMarkedReviewSummaries() throws IOException { List requests = new ArrayList<>(); @@ -253,4 +289,43 @@ void clearPreviousReviewBodies_clearsOnlyMarkedReviewSummaries() throws IOExcept assertThat(new ObjectMapper().readTree(body.readUtf8()).path("body").asText()) .isEqualTo(""); } + + @Test + void clearPreviousReviewBodies_preservesTheReplacementReview() throws IOException { + List requests = new ArrayList<>(); + when(okHttpClient.newCall(any(Request.class))).thenAnswer(invocation -> { + Request request = invocation.getArgument(0); + requests.add(request); + + String responseJson = request.method().equals("GET") + ? "[{\"id\":51,\"body\":\"old \"}," + + "{\"id\":99,\"body\":\"new \"}]" + : "{}"; + Response requestResponse = new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create(responseJson, MediaType.parse("application/json"))) + .build(); + Call requestCall = mock(Call.class); + when(requestCall.execute()).thenReturn(requestResponse); + return requestCall; + }); + + int cleared = action.clearPreviousReviewBodies( + "owner", + "repo", + 123, + "", + "", + 99L); + + assertThat(cleared).isEqualTo(1); + assertThat(requests).extracting(request -> request.method() + " " + request.url().encodedPath()) + .containsExactly( + "GET /repos/owner/repo/pulls/123/reviews", + "PUT /repos/owner/repo/pulls/123/reviews/51" + ); + } } diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffActionTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffActionTest.java index 3a1b2df41..d2ece66aa 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffActionTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/github/actions/GetPullRequestDiffActionTest.java @@ -8,6 +8,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -102,4 +103,42 @@ void testGetPullRequestDiff_UnsuccessfulResponse_ThrowsIOException() throws IOEx verify(response).close(); } + + @Test + void getPullRequestFilePatchesReturnsStructuredDestinationPaths() throws IOException { + String filesJson = """ + [ + { + "filename": "src/New Name.java", + "previous_filename": "src/Old Name.java", + "status": "renamed", + "patch": "@@ -1 +1 @@\\n-old\\n+new" + }, + { + "filename": "assets/logo.png", + "status": "modified" + } + ] + """; + + when(okHttpClient.newCall(any(Request.class))).thenReturn(call); + when(call.execute()).thenReturn(response); + when(response.isSuccessful()).thenReturn(true); + when(response.body()).thenReturn(responseBody); + when(responseBody.string()).thenReturn(filesJson); + when(response.header("Link")).thenReturn(null); + + List patches = + action.getPullRequestFilePatches("owner", "repo", 123); + + assertThat(patches).containsExactly( + new GetPullRequestDiffAction.PullRequestFilePatch( + "src/New Name.java", + "src/Old Name.java", + "renamed", + "@@ -1 +1 @@\n-old\n+new"), + new GetPullRequestDiffAction.PullRequestFilePatch( + "assets/logo.png", "", "modified", "")); + verify(response).close(); + } } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java index 962db55e9..a8b330a24 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/bitbucket/webhookhandler/BitbucketCloudBranchWebhookHandler.java @@ -167,7 +167,8 @@ public WebhookResult handle(WebhookPayload payload, Project project, Consumer result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); return WebhookResult.success("Branch analysis completed", result); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java index a68281fea..5f6b42288 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfig.java @@ -2,6 +2,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -42,10 +43,12 @@ public Executor taskExecutor() { * Sized for concurrent webhook handling from VCS providers. */ @Bean(name = "webhookExecutor") - public Executor webhookExecutor() { + public Executor webhookExecutor( + @Value("${webhook.executor.core-pool-size:8}") int corePoolSize, + @Value("${webhook.executor.max-pool-size:20}") int maxPoolSize) { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(4); - executor.setMaxPoolSize(8); + executor.setCorePoolSize(corePoolSize); + executor.setMaxPoolSize(maxPoolSize); // Do not accept work into an ephemeral in-memory backlog. Saturated work // remains QUEUED in the database and is retried by the recovery scheduler. executor.setQueueCapacity(0); @@ -55,7 +58,7 @@ public Executor webhookExecutor() { executor.setRejectedExecutionHandler(new java.util.concurrent.ThreadPoolExecutor.AbortPolicy()); executor.initialize(); log.info("Webhook executor initialized with core={}, max={}, durable database backlog enabled", - 4, 8); + corePoolSize, maxPoolSize); return executor; } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/ProviderPipelineActionController.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/ProviderPipelineActionController.java index 94924d1ce..1754dd4fc 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/ProviderPipelineActionController.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/controller/ProviderPipelineActionController.java @@ -76,7 +76,8 @@ public ResponseEntity handlePrWebhook( PipelineActionProcessor.EventConsumer dualConsumer = pipelineJobService.createDualConsumer(jobRef, consumer); try { - return pipelineActionProcessor.processPipelineActionWithConsumer(payload, dualConsumer); + return pipelineActionProcessor.processPipelineActionWithConsumer( + payload, dualConsumer, jobRef); } catch (org.rostilos.codecrow.analysisengine.exception.AnalysisLockedException e) { log.warn("Analysis locked: {}", e.getMessage()); dualConsumer.accept(Map.of( @@ -131,7 +132,8 @@ public ResponseEntity handleBranchWebhook( PipelineActionProcessor.EventConsumer dualConsumer = pipelineJobService.createDualConsumer(jobRef, consumer); try { - return pipelineActionProcessor.processPipelineActionWithConsumer(payload, dualConsumer); + return pipelineActionProcessor.processPipelineActionWithConsumer( + payload, dualConsumer, jobRef); } catch (org.rostilos.codecrow.analysisengine.exception.AnalysisLockedException e) { log.warn("Analysis locked: {}", e.getMessage()); dualConsumer.accept(Map.of( diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessor.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessor.java index b725a5d25..06d9eb44e 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessor.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessor.java @@ -9,6 +9,8 @@ import org.rostilos.codecrow.analysisengine.processor.analysis.BranchAnalysisProcessor; import org.rostilos.codecrow.analysisengine.processor.analysis.PullRequestAnalysisProcessor; import org.rostilos.codecrow.analysisengine.service.ProjectValidationService; +import org.rostilos.codecrow.analysisengine.service.branch.BranchAnalysisGateService; +import org.rostilos.codecrow.core.model.job.Job; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -30,15 +32,18 @@ public class PipelineActionProcessor { private final ProjectValidationService projectService; private final PullRequestAnalysisProcessor pullRequestAnalysisProcessor; private final BranchAnalysisProcessor branchAnalysisProcessor; + private final BranchAnalysisGateService branchAnalysisGateService; public PipelineActionProcessor( ProjectValidationService projectService, PullRequestAnalysisProcessor pullRequestAnalysisProcessor, - BranchAnalysisProcessor branchAnalysisProcessor + BranchAnalysisProcessor branchAnalysisProcessor, + BranchAnalysisGateService branchAnalysisGateService ) { this.projectService = projectService; this.pullRequestAnalysisProcessor = pullRequestAnalysisProcessor; this.branchAnalysisProcessor = branchAnalysisProcessor; + this.branchAnalysisGateService = branchAnalysisGateService; } public interface EventConsumer { @@ -56,12 +61,36 @@ public Map processPipelineActionWithConsumer( @Valid @RequestBody AnalysisProcessRequest request, EventConsumer consumer ) throws GeneralSecurityException { + return processPipelineActionWithConsumer(request, consumer, null); + } + + public Map processPipelineActionWithConsumer( + AnalysisProcessRequest request, + EventConsumer consumer, + Job job + ) throws GeneralSecurityException { try { Project project = projectService.getProjectWithConnections(request.getProjectId()); + boolean dependenciesGated = job != null; + if (dependenciesGated) { + BranchAnalysisGateService.GateResult gateResult = + branchAnalysisGateService.awaitDependencies( + project.getId(), job, consumer::accept); + if (gateResult == BranchAnalysisGateService.GateResult.SUPERSEDED) { + return Map.of( + "status", "ignored", + "message", "Superseded by a newer branch analysis job"); + } + } if(request.getAnalysisType() == AnalysisType.BRANCH_ANALYSIS) { - return branchAnalysisProcessor.process((BranchProcessRequest) request, consumer::accept); + if (dependenciesGated) { + return branchAnalysisProcessor.processAfterDependencyGate( + (BranchProcessRequest) request, consumer::accept); + } + return branchAnalysisProcessor.process( + (BranchProcessRequest) request, consumer::accept); } else { return pullRequestAnalysisProcessor.process( (PrProcessRequest) request, diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessor.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessor.java index 67f7733c3..45f074baf 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessor.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessor.java @@ -144,13 +144,13 @@ public void processWebhookInTransaction( jobService.startJob(job); log.info("jobService.startJob completed for job {}", job.getExternalId()); - if (job.getJobType() == JobType.BRANCH_ANALYSIS) { - BranchAnalysisGateService.GateResult gateResult = branchAnalysisGateService.awaitTurn( - projectId, - job.getBranchName(), - job.getId(), - job.getPrNumber(), - event -> logHandlerEvent(job, event)); + if (job.getJobType() == JobType.BRANCH_ANALYSIS + || job.getJobType() == JobType.PR_ANALYSIS) { + BranchAnalysisGateService.GateResult gateResult = + branchAnalysisGateService.awaitDependencies( + projectId, + job, + event -> logHandlerEvent(job, event)); if (gateResult == BranchAnalysisGateService.GateResult.SUPERSEDED) { String reason = "Superseded by a newer branch analysis job for " + job.getBranchName(); log.info("Skipping branch job {}: {}", job.getExternalId(), reason); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingService.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingService.java index f73b7475d..b93baad1d 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingService.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingService.java @@ -13,6 +13,7 @@ import org.rostilos.codecrow.vcsclient.bitbucket.service.ReportGenerator; import org.rostilos.codecrow.vcsclient.github.actions.CheckRunAction; import org.rostilos.codecrow.vcsclient.github.actions.CommentOnPullRequestAction; +import org.rostilos.codecrow.vcsclient.github.actions.GetPullRequestDiffAction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -103,23 +104,29 @@ public void postAnalysisResults( pullRequestNumber, placeholderCommentId); AnalysisSummary summary = reportGenerator.createAnalysisSummary(codeAnalysis, platformPrEntityId); + VcsRepoInfo vcsRepoInfo = getVcsRepoInfo(project); + OkHttpClient httpClient = vcsClientProvider.getHttpClient( + vcsRepoInfo.getVcsConnection() + ); + GitHubReviewFormatter.ReviewPlan reviewPlan = createReviewPlan( + httpClient, vcsRepoInfo, pullRequestNumber, summary); + // Use GitHub-specific markdown with collapsible spoilers for suggested fixes String markdownSummary = reportGenerator.createMarkdownSummary(codeAnalysis, summary, true); + String nonInlineFindingsMarkdown = reviewFormatter.formatNonInlineFindings( + reviewPlan.nonInlineFindings()); String detailedIssuesMarkdown = reportGenerator.createDetailedIssuesMarkdown(summary, true); // GitHub doesn't support threaded replies for issue comments like Bitbucket does. // So we combine summary and detailed issues into ONE comment. String fullComment = markdownSummary; + if (!nonInlineFindingsMarkdown.isEmpty()) { + fullComment += "\n\n---\n\n" + nonInlineFindingsMarkdown; + } if (detailedIssuesMarkdown != null && !detailedIssuesMarkdown.isEmpty()) { - fullComment = markdownSummary + "\n\n---\n\n" + detailedIssuesMarkdown; + fullComment += "\n\n---\n\n" + detailedIssuesMarkdown; } - VcsRepoInfo vcsRepoInfo = getVcsRepoInfo(project); - - OkHttpClient httpClient = vcsClientProvider.getHttpClient( - vcsRepoInfo.getVcsConnection() - ); - // Post or update comment with full content (summary + issues) if (placeholderCommentId != null) { updatePlaceholderComment(httpClient, vcsRepoInfo, pullRequestNumber, fullComment, placeholderCommentId); @@ -129,13 +136,44 @@ public void postAnalysisResults( // Publish findings as a submitted COMMENT review so they appear inline in // "Files changed" and as grouped reviewer comments in the PR conversation. - postInlineReviewComments(httpClient, vcsRepoInfo, pullRequestNumber, codeAnalysis, summary); + postInlineReviewComments( + httpClient, vcsRepoInfo, pullRequestNumber, codeAnalysis, reviewPlan); // Create Check Run for the commit createCheckRun(httpClient, vcsRepoInfo, codeAnalysis, summary); log.info("Successfully posted analysis results to GitHub"); } + + private GitHubReviewFormatter.ReviewPlan createReviewPlan( + OkHttpClient httpClient, + VcsRepoInfo vcsRepoInfo, + Long pullRequestNumber, + AnalysisSummary summary + ) { + try { + List filePatches = + new GetPullRequestDiffAction(httpClient).getPullRequestFilePatches( + vcsRepoInfo.getRepoWorkspace(), + vcsRepoInfo.getRepoSlug(), + pullRequestNumber.intValue()); + GitHubReviewFormatter.ReviewPlan plan = reviewFormatter.planComments( + summary.getIssues(), CODECROW_REVIEW_MARKER, filePatches); + log.info("Prepared GitHub review plan for PR {}: {} inline, {} not inline", + pullRequestNumber, + plan.inlineComments().size(), + plan.nonInlineFindings().size()); + return plan; + } catch (Exception e) { + // Diff enrichment is supplemental. Publishing an unverified anchor can + // make GitHub reject the whole review, so retain every finding in the + // aggregate comment instead of guessing. + log.warn("Could not load GitHub PR diff for inline comment planning on PR {}: {}. " + + "Findings will remain in the aggregate comment.", + pullRequestNumber, e.getMessage()); + return reviewFormatter.planWithoutDiff(summary.getIssues()); + } + } /** * Post summary as a regular comment. @@ -201,15 +239,14 @@ private void postInlineReviewComments( VcsRepoInfo vcsRepoInfo, Long pullRequestNumber, CodeAnalysis codeAnalysis, - AnalysisSummary summary + GitHubReviewFormatter.ReviewPlan reviewPlan ) { CommentOnPullRequestAction commentAction = new CommentOnPullRequestAction(httpClient); - cleanupPreviousInlineReviewComments(commentAction, vcsRepoInfo, pullRequestNumber); - - List> comments = reviewFormatter.formatComments( - summary.getIssues(), CODECROW_REVIEW_MARKER); + List> comments = reviewPlan.inlineComments(); if (comments.isEmpty()) { - log.debug("No confidently anchored issues to post as GitHub review comments"); + cleanupPreviousInlineReviewComments( + commentAction, vcsRepoInfo, pullRequestNumber, null); + log.debug("No diff-verified issues to post as GitHub review comments"); return; } @@ -221,7 +258,7 @@ private void postInlineReviewComments( } try { - commentAction.createPullRequestReview( + String reviewId = commentAction.createPullRequestReview( vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(), pullRequestNumber.intValue(), @@ -232,27 +269,53 @@ private void postInlineReviewComments( ); log.info("Posted GitHub review with {} inline comment(s) on PR {}", comments.size(), pullRequestNumber); + Long preservedReviewId = parseReviewId(reviewId); + if (preservedReviewId == null) { + log.warn("GitHub created the replacement review for PR {} without a usable ID; " + + "leaving previous review artifacts intact", + pullRequestNumber); + } else { + cleanupPreviousInlineReviewComments( + commentAction, + vcsRepoInfo, + pullRequestNumber, + preservedReviewId); + } } catch (Exception e) { // Inline review publishing is supplemental. The aggregate comment - // still contains the complete result, and the Check Run still carries - // the quality status when GitHub rejects a stale or non-diff anchor. + // still contains the complete result. Cleanup deliberately happens + // only after a replacement succeeds, so a rejected review does not + // erase the last successfully published inline comments. log.warn("Failed to post inline GitHub review comments on PR {}: {}. " + "Issues remain available in the summary comment.", pullRequestNumber, e.getMessage()); } } + private Long parseReviewId(String reviewId) { + if (reviewId == null || reviewId.isBlank()) { + return null; + } + try { + return Long.parseLong(reviewId); + } catch (NumberFormatException e) { + return null; + } + } + private void cleanupPreviousInlineReviewComments( CommentOnPullRequestAction commentAction, VcsRepoInfo vcsRepoInfo, - Long pullRequestNumber + Long pullRequestNumber, + Long preservedReviewId ) { try { int deleted = commentAction.deletePreviousReviewComments( vcsRepoInfo.getRepoWorkspace(), vcsRepoInfo.getRepoSlug(), pullRequestNumber.intValue(), - CODECROW_REVIEW_MARKER + CODECROW_REVIEW_MARKER, + preservedReviewId ); if (deleted > 0) { log.info("Deleted {} previous CodeCrow inline review comment(s) from PR {}", @@ -271,7 +334,8 @@ private void cleanupPreviousInlineReviewComments( vcsRepoInfo.getRepoSlug(), pullRequestNumber.intValue(), CODECROW_REVIEW_MARKER, - CODECROW_CLEARED_REVIEW_MARKER + CODECROW_CLEARED_REVIEW_MARKER, + preservedReviewId ); if (cleared > 0) { log.info("Cleared {} previous CodeCrow review summary body/bodies from PR {}", diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatter.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatter.java index 17266831d..af897d3b4 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatter.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatter.java @@ -2,36 +2,60 @@ import org.rostilos.codecrow.core.util.tracking.DiffSanitizer; import org.rostilos.codecrow.vcsclient.bitbucket.model.report.AnalysisSummary; +import org.rostilos.codecrow.vcsclient.github.actions.GetPullRequestDiffAction.PullRequestFilePatch; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Formats CodeCrow issues for GitHub's pull-request review API. */ final class GitHubReviewFormatter { private static final int MAX_INLINE_COMMENTS = 20; + private static final Pattern HUNK_HEADER = Pattern.compile( + "^@@ -\\d+(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@.*$"); + private static final String OUTSIDE_DIFF_REASON = + "The reported line is outside the current pull-request diff."; + private static final String DIFF_UNAVAILABLE_REASON = + "The current pull-request diff could not be loaded, so CodeCrow did not guess an inline anchor."; - List> formatComments( + ReviewPlan planComments( List issues, - String marker + String marker, + List filePatches ) { if (issues == null || issues.isEmpty()) { - return List.of(); + return ReviewPlan.empty(); } + Map> rightSideLines = collectRightSideLines(filePatches); List> comments = new ArrayList<>(); + List nonInlineFindings = new ArrayList<>(); for (AnalysisSummary.IssueSummary issue : issues) { - if (comments.size() >= MAX_INLINE_COMMENTS) { - break; - } - String path = normalizePath(issue.getFilePath()); Integer line = issue.getLineNumber(); - if (path == null || line == null || line <= 0 || !hasConfidentAnchor(issue)) { + String ineligibleReason = ineligibleReason(issue, path, line); + if (ineligibleReason != null) { + nonInlineFindings.add(new NonInlineFinding(issue, ineligibleReason)); + continue; + } + if (!rightSideLines.getOrDefault(path, Set.of()).contains(line)) { + nonInlineFindings.add(new NonInlineFinding(issue, OUTSIDE_DIFF_REASON)); + continue; + } + if (comments.size() >= MAX_INLINE_COMMENTS) { + nonInlineFindings.add(new NonInlineFinding( + issue, + "GitHub's limit of " + MAX_INLINE_COMMENTS + + " CodeCrow inline comments was reached.")); continue; } @@ -42,7 +66,52 @@ List> formatComments( comment.put("body", formatBody(issue, marker)); comments.add(comment); } - return List.copyOf(comments); + return new ReviewPlan(comments, nonInlineFindings); + } + + ReviewPlan planWithoutDiff(List issues) { + if (issues == null || issues.isEmpty()) { + return ReviewPlan.empty(); + } + return new ReviewPlan( + List.of(), + issues.stream() + .map(issue -> new NonInlineFinding(issue, DIFF_UNAVAILABLE_REASON)) + .toList()); + } + + String formatNonInlineFindings(List findings) { + if (findings == null || findings.isEmpty()) { + return ""; + } + + StringBuilder markdown = new StringBuilder(); + markdown.append("
\n📍 Findings not posted inline (") + .append(findings.size()) + .append(")\n\n") + .append("GitHub only accepts inline review comments on lines available in the current pull-request diff. ") + .append("These findings remain part of the complete review.\n\n"); + + for (NonInlineFinding finding : findings) { + AnalysisSummary.IssueSummary issue = finding.issue(); + String title = issue.getTitle() == null || issue.getTitle().isBlank() + ? "Untitled finding" + : issue.getTitle(); + markdown.append("- ").append(severityEmoji(issue)).append(" **") + .append(issue.getSeverity()).append("** — "); + if (issue.getIssueUrl() != null && !issue.getIssueUrl().isBlank()) { + markdown.append("[").append(title).append("](") + .append(issue.getIssueUrl()).append(")"); + } else { + markdown.append("**").append(title).append("**"); + } + markdown.append(" at `") + .append(issue.getLocationDescription().replace("`", "\\`")) + .append("`\n - ").append(finding.reason()).append("\n"); + } + + markdown.append("\n
"); + return markdown.toString(); } String formatReviewBody(int commentCount, String marker) { @@ -58,6 +127,68 @@ private boolean hasConfidentAnchor(AnalysisSummary.IssueSummary issue) { || (issue.getCodeSnippet() != null && !issue.getCodeSnippet().isBlank()); } + private String ineligibleReason( + AnalysisSummary.IssueSummary issue, + String path, + Integer line + ) { + if (path == null) { + return "No repository file path was available for an inline anchor."; + } + if (line == null || line <= 0) { + return "No positive source line was available for an inline anchor."; + } + if (!hasConfidentAnchor(issue)) { + return "The finding does not have a confident source-line anchor."; + } + return null; + } + + private Map> collectRightSideLines( + List filePatches + ) { + Map> rightSideLines = new HashMap<>(); + if (filePatches == null) { + return rightSideLines; + } + + for (PullRequestFilePatch filePatch : filePatches) { + String path = normalizePath(filePatch.filename()); + if (path == null || filePatch.patch().isBlank()) { + continue; + } + Set lines = rightSideLines.computeIfAbsent( + path, ignored -> new HashSet<>()); + collectRightSideLines(filePatch.patch(), lines); + } + return rightSideLines; + } + + private void collectRightSideLines(String patch, Set lines) { + int nextRightLine = -1; + boolean inHunk = false; + for (String diffLine : patch.split("\\r?\\n", -1)) { + Matcher hunk = HUNK_HEADER.matcher(diffLine); + if (hunk.matches()) { + nextRightLine = Integer.parseInt(hunk.group(1)); + inHunk = true; + continue; + } + if (!inHunk || diffLine.isEmpty() || diffLine.startsWith("\\")) { + continue; + } + + char prefix = diffLine.charAt(0); + if (prefix == '+') { + lines.add(nextRightLine++); + } else if (prefix == ' ') { + lines.add(nextRightLine++); + } else if (prefix != '-') { + inHunk = false; + } + } + } + private String normalizePath(String path) { if (path == null || path.isBlank()) { return null; @@ -120,6 +251,9 @@ private void appendSuggestedFix( } private String severityEmoji(AnalysisSummary.IssueSummary issue) { + if (issue.getSeverity() == null) { + return "ℹ️"; + } return switch (issue.getSeverity()) { case HIGH -> "🔴"; case MEDIUM -> "🟡"; @@ -145,4 +279,23 @@ private String humanizeCategory(String category) { } return result.toString(); } + + record ReviewPlan( + List> inlineComments, + List nonInlineFindings + ) { + ReviewPlan { + inlineComments = inlineComments == null ? List.of() : List.copyOf(inlineComments); + nonInlineFindings = nonInlineFindings == null + ? List.of() + : List.copyOf(nonInlineFindings); + } + + static ReviewPlan empty() { + return new ReviewPlan(List.of(), List.of()); + } + } + + record NonInlineFinding(AnalysisSummary.IssueSummary issue, String reason) { + } } diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubBranchWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubBranchWebhookHandler.java index 44a60c659..6366380f3 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubBranchWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubBranchWebhookHandler.java @@ -100,7 +100,8 @@ public WebhookResult handle(WebhookPayload payload, Project project, Consumer result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); return WebhookResult.success("Branch analysis completed", result); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java index b4d70fffe..80ad0679b 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/github/webhookhandler/GitHubPullRequestWebhookHandler.java @@ -370,7 +370,8 @@ private WebhookResult handlePrMergeEvent( } }; - Map result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); return WebhookResult.success("Branch reconciliation completed after PR #" + prNumber + " merge", result); diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabBranchWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabBranchWebhookHandler.java index ad21e1244..371643b33 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabBranchWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabBranchWebhookHandler.java @@ -109,7 +109,8 @@ private WebhookResult handlePushEvent( }; // Delegate to branch analysis processor - Map result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); // Check if analysis failed if ("error".equals(result.get("status"))) { diff --git a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java index 2363e3857..51993f7dc 100644 --- a/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java +++ b/java-ecosystem/services/pipeline-agent/src/main/java/org/rostilos/codecrow/pipelineagent/gitlab/webhookhandler/GitLabMrMergeWebhookHandler.java @@ -161,7 +161,8 @@ private WebhookResult handleMrMergeEvent( } }; - Map result = branchAnalysisProcessor.process(request, processorConsumer); + Map result = branchAnalysisProcessor.processAfterDependencyGate( + request, processorConsumer); return WebhookResult.success("Branch reconciliation completed after MR !" + mrNumber + " merge", result); diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java new file mode 100644 index 000000000..639424298 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/config/AsyncConfigTest.java @@ -0,0 +1,40 @@ +package org.rostilos.codecrow.pipelineagent.config; + +import org.junit.jupiter.api.Test; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +class AsyncConfigTest { + + @Test + void webhookExecutorRunsFiveAcceptedReviewsInParallel() throws Exception { + Executor configured = new AsyncConfig().webhookExecutor(2, 5); + ThreadPoolTaskExecutor executor = (ThreadPoolTaskExecutor) configured; + CountDownLatch started = new CountDownLatch(5); + CountDownLatch release = new CountDownLatch(1); + + try { + for (int i = 0; i < 5; i++) { + executor.execute(() -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + }); + } + + assertThat(started.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(executor.getMaxPoolSize()).isEqualTo(5); + } finally { + release.countDown(); + executor.shutdown(); + } + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessorDependencyGateTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessorDependencyGateTest.java new file mode 100644 index 000000000..e1cd64bd7 --- /dev/null +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/PipelineActionProcessorDependencyGateTest.java @@ -0,0 +1,103 @@ +package org.rostilos.codecrow.pipelineagent.generic.processor; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; +import org.rostilos.codecrow.analysisengine.dto.request.processor.PrProcessRequest; +import org.rostilos.codecrow.analysisengine.processor.analysis.BranchAnalysisProcessor; +import org.rostilos.codecrow.analysisengine.processor.analysis.PullRequestAnalysisProcessor; +import org.rostilos.codecrow.analysisengine.service.ProjectValidationService; +import org.rostilos.codecrow.analysisengine.service.branch.BranchAnalysisGateService; +import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobType; +import org.rostilos.codecrow.core.model.project.Project; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class PipelineActionProcessorDependencyGateTest { + + @Mock private ProjectValidationService projectService; + @Mock private PullRequestAnalysisProcessor pullRequestAnalysisProcessor; + @Mock private BranchAnalysisProcessor branchAnalysisProcessor; + @Mock private BranchAnalysisGateService branchAnalysisGateService; + @Mock private Project project; + + private PipelineActionProcessor processor; + + @BeforeEach + void setUp() throws Exception { + processor = new PipelineActionProcessor( + projectService, + pullRequestAnalysisProcessor, + branchAnalysisProcessor, + branchAnalysisGateService); + when(projectService.getProjectWithConnections(1L)).thenReturn(project); + when(project.getId()).thenReturn(1L); + } + + @Test + void pipelinePrResolvesDurableDependenciesBeforeAnalysis() throws Exception { + PrProcessRequest request = new PrProcessRequest(); + request.projectId = 1L; + request.analysisType = AnalysisType.PR_REVIEW; + Job job = new Job(); + job.setJobType(JobType.PR_ANALYSIS); + + when(branchAnalysisGateService.awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(job), + any())).thenReturn(BranchAnalysisGateService.GateResult.READY); + when(pullRequestAnalysisProcessor.process(any(), any(), any())) + .thenReturn(Map.of("status", "accepted")); + + Map result = processor.processPipelineActionWithConsumer( + request, event -> { }, job); + + assertThat(result).containsEntry("status", "accepted"); + InOrder ordered = inOrder(branchAnalysisGateService, pullRequestAnalysisProcessor); + ordered.verify(branchAnalysisGateService).awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(job), + any()); + ordered.verify(pullRequestAnalysisProcessor).process( + org.mockito.ArgumentMatchers.eq(request), any(), + org.mockito.ArgumentMatchers.eq(project)); + } + + @Test + void pipelineBranchSkipsTheProcessorFallbackAfterDurableGate() throws Exception { + BranchProcessRequest request = new BranchProcessRequest(); + request.projectId = 1L; + request.analysisType = AnalysisType.BRANCH_ANALYSIS; + Job job = new Job(); + job.setJobType(JobType.BRANCH_ANALYSIS); + + when(branchAnalysisGateService.awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(job), + any())).thenReturn(BranchAnalysisGateService.GateResult.READY); + when(branchAnalysisProcessor.processAfterDependencyGate(any(), any())) + .thenReturn(Map.of("status", "accepted")); + + Map result = processor.processPipelineActionWithConsumer( + request, event -> { }, job); + + assertThat(result).containsEntry("status", "accepted"); + verify(branchAnalysisProcessor).processAfterDependencyGate( + org.mockito.ArgumentMatchers.eq(request), any()); + verify(branchAnalysisProcessor, never()).process(any(), any()); + } +} diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessorBranchGateTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessorBranchGateTest.java index 13e561ecf..f4737fcac 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessorBranchGateTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/generic/processor/WebhookAsyncProcessorBranchGateTest.java @@ -72,11 +72,9 @@ void supersededBranchJobNeverInvokesProviderHandler() { "41", "feature/one", "main", "merge-1", null); when(projectRepository.findById(1L)).thenReturn(Optional.of(project)); - when(branchAnalysisGateService.awaitTurn( + when(branchAnalysisGateService.awaitDependencies( org.mockito.ArgumentMatchers.eq(1L), - org.mockito.ArgumentMatchers.eq("main"), - org.mockito.ArgumentMatchers.eq(101L), - org.mockito.ArgumentMatchers.eq(41L), + org.mockito.ArgumentMatchers.eq(branchJob), any())) .thenReturn(BranchAnalysisGateService.GateResult.SUPERSEDED); @@ -92,6 +90,39 @@ void supersededBranchJobNeverInvokesProviderHandler() { verify(ragOperationsService).deletePrFiles(project, 41); } + @Test + void prJobPassesTargetBranchDependencyGateBeforeProviderHandler() { + Job prJob = new Job(); + ReflectionTestUtils.setField(prJob, "id", 102L); + prJob.setProject(project); + prJob.setJobType(JobType.PR_ANALYSIS); + prJob.setBranchName("main"); + prJob.setPrNumber(42L); + + WebhookPayload payload = new WebhookPayload( + EVcsProvider.GITHUB, "pull_request", "repo-id", "repo", "owner", + "42", "feature/two", "main", "head-2", null); + WebhookHandler.WebhookResult success = WebhookHandler.WebhookResult.ignored( + "test dependency ordering"); + + when(projectRepository.findById(1L)).thenReturn(Optional.of(project)); + when(branchAnalysisGateService.awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(prJob), + any())).thenReturn(BranchAnalysisGateService.GateResult.READY); + when(handler.handle(any(), any(), any())).thenReturn(success); + + processor.processWebhookInTransaction( + EVcsProvider.GITHUB, 1L, payload, handler, prJob); + + var ordered = org.mockito.Mockito.inOrder(branchAnalysisGateService, handler); + ordered.verify(branchAnalysisGateService).awaitDependencies( + org.mockito.ArgumentMatchers.eq(1L), + org.mockito.ArgumentMatchers.eq(prJob), + any()); + ordered.verify(handler).handle(any(), any(), any()); + } + @Test void promptDryRunSuppressesEveryWebhookVcsMutationPath() { System.setProperty(PromptDryRunMode.ENABLED_KEY, "true"); diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingServiceTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingServiceTest.java index 5d8c33863..8b7d524c8 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingServiceTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReportingServiceTest.java @@ -50,6 +50,7 @@ class GitHubReportingServiceTest { private CodeAnalysis analysis; private Project project; private AnalysisSummary summary; + private AnalysisSummary.IssueSummary inlineIssue; @BeforeEach void setUp() { @@ -68,7 +69,7 @@ void setUp() { when(repoInfo.getVcsConnection()).thenReturn(connection); org.mockito.Mockito.lenient().when(analysis.getCommitHash()).thenReturn("head-sha"); - AnalysisSummary.IssueSummary issue = new AnalysisSummary.IssueSummary( + inlineIssue = new AnalysisSummary.IssueSummary( IssueSeverity.HIGH, "SECURITY", "src/App.java", @@ -81,7 +82,7 @@ void setUp() { 7L, "return repository.findById(id);" ); - org.mockito.Mockito.lenient().when(summary.getIssues()).thenReturn(List.of(issue)); + org.mockito.Mockito.lenient().when(summary.getIssues()).thenReturn(List.of(inlineIssue)); org.mockito.Mockito.lenient().when(summary.getTotalUnresolvedIssues()).thenReturn(1); org.mockito.Mockito.lenient().when(summary.getFileIssueCount()).thenReturn(Map.of()); org.mockito.Mockito.lenient().when(reportGenerator.createAnalysisSummary(analysis, 77L)).thenReturn(summary); @@ -125,7 +126,82 @@ void preservesAggregateCommentAndAddsSubmittedInlineReview() throws IOException } @Test - void removesPreviousGeneratedReviewArtifactsBeforePostingReplacement() throws IOException { + void postsValidCommentsAndAddsOutOfDiffFindingsToTheAggregateSpoiler() + throws IOException { + AnalysisSummary.IssueSummary outsideDiff = new AnalysisSummary.IssueSummary( + IssueSeverity.MEDIUM, + "TESTING", + "src/App.java", + 1, + "File-wide constructor mismatch", + "The reported location is not part of the pull-request patch.", + null, + null, + "https://codecrow.example/issues/8", + 8L, + "service = new AppService();" + ); + when(summary.getIssues()).thenReturn(List.of(inlineIssue, outsideDiff)); + + List requests = new ArrayList<>(); + when(vcsClientProvider.getHttpClient(org.mockito.ArgumentMatchers.any())) + .thenReturn(capturingClient(requests, false, false)); + + service.postAnalysisResults(analysis, project, 42L, 77L, "99"); + + CapturedRequest aggregate = requestAt(requests, "/repos/owner/repo/issues/comments/99"); + String aggregateBody = OBJECT_MAPPER.readTree(aggregate.body()).path("body").asText(); + assertThat(aggregateBody) + .contains("📍 Findings not posted inline (1)") + .contains("[File-wide constructor mismatch](https://codecrow.example/issues/8)") + .contains("outside the current pull-request diff"); + + CapturedRequest review = requestAt( + requests, "POST", "/repos/owner/repo/pulls/42/reviews"); + JsonNode reviewPayload = OBJECT_MAPPER.readTree(review.body()); + assertThat(reviewPayload.path("comments")).hasSize(1); + assertThat(reviewPayload.path("comments").get(0).path("line").asInt()).isEqualTo(12); + } + + @Test + void allOutOfDiffFindingsStayInTheAggregateAndRemoveStaleInlineArtifacts() + throws IOException { + AnalysisSummary.IssueSummary outsideDiff = new AnalysisSummary.IssueSummary( + IssueSeverity.MEDIUM, + "TESTING", + "src/App.java", + 1, + "File-wide constructor mismatch", + "The reported location is not part of the pull-request patch.", + null, + null, + "https://codecrow.example/issues/8", + 8L, + "service = new AppService();" + ); + when(summary.getIssues()).thenReturn(List.of(outsideDiff)); + + List requests = new ArrayList<>(); + when(vcsClientProvider.getHttpClient(org.mockito.ArgumentMatchers.any())) + .thenReturn(capturingClient(requests, false, true)); + + service.postAnalysisResults(analysis, project, 42L, 77L, "99"); + + assertThat(requests).noneMatch(request -> request.method().equals("POST") + && request.path().equals("/repos/owner/repo/pulls/42/reviews")); + assertThat(indexOf(requests, "DELETE", "/repos/owner/repo/pulls/comments/321")) + .isGreaterThanOrEqualTo(0); + assertThat(indexOf(requests, "PUT", "/repos/owner/repo/pulls/42/reviews/654")) + .isGreaterThanOrEqualTo(0); + + CapturedRequest aggregate = requestAt(requests, "/repos/owner/repo/issues/comments/99"); + assertThat(OBJECT_MAPPER.readTree(aggregate.body()).path("body").asText()) + .contains("Findings not posted inline (1)") + .contains("File-wide constructor mismatch"); + } + + @Test + void removesPreviousGeneratedReviewArtifactsAfterPostingReplacement() throws IOException { List requests = new ArrayList<>(); when(vcsClientProvider.getHttpClient(org.mockito.ArgumentMatchers.any())) .thenReturn(capturingClient(requests, false, true)); @@ -137,17 +213,17 @@ void removesPreviousGeneratedReviewArtifactsBeforePostingReplacement() throws IO int reviewIndex = indexOf(requests, "POST", "/repos/owner/repo/pulls/42/reviews"); assertThat(deleteIndex).isGreaterThanOrEqualTo(0); assertThat(clearIndex).isGreaterThan(deleteIndex); - assertThat(reviewIndex).isGreaterThan(deleteIndex); - assertThat(reviewIndex).isGreaterThan(clearIndex); + assertThat(reviewIndex).isLessThan(deleteIndex); + assertThat(reviewIndex).isLessThan(clearIndex); assertThat(OBJECT_MAPPER.readTree(requests.get(clearIndex).body()).path("body").asText()) .isEqualTo(""); } @Test - void reviewRejectionDoesNotBlockTheSummaryOrCheckRun() { + void reviewRejectionDoesNotBlockTheSummaryOrCheckRunOrDeletePreviousComments() { List requests = new ArrayList<>(); when(vcsClientProvider.getHttpClient(org.mockito.ArgumentMatchers.any())) - .thenReturn(capturingClient(requests, true, false)); + .thenReturn(capturingClient(requests, true, true)); assertThatCode(() -> service.postAnalysisResults(analysis, project, 42L, 77L, "99")) .doesNotThrowAnyException(); @@ -158,6 +234,9 @@ void reviewRejectionDoesNotBlockTheSummaryOrCheckRun() { "/repos/owner/repo/pulls/42/reviews", "/repos/owner/repo/check-runs" ); + assertThat(requests).noneMatch(request -> request.method().equals("DELETE")); + assertThat(requests).noneMatch(request -> request.method().equals("PUT") + && request.path().equals("/repos/owner/repo/pulls/42/reviews/654")); } @Test @@ -206,15 +285,22 @@ private OkHttpClient capturingClient( requests.add(new CapturedRequest(request.method(), path, buffer.readUtf8())); boolean reviewPost = request.method().equals("POST") && path.endsWith("/reviews"); + boolean pullRequestFiles = request.method().equals("GET") + && path.endsWith("/pulls/42/files"); boolean reviewCommentList = request.method().equals("GET") && path.endsWith("/pulls/42/comments"); boolean reviewList = request.method().equals("GET") && path.endsWith("/pulls/42/reviews"); boolean rejected = rejectReviews && reviewPost; String responseJson; - if (reviewCommentList) { + if (pullRequestFiles) { + responseJson = "[{\"filename\":\"src/App.java\"," + + "\"status\":\"modified\"," + + "\"patch\":\"@@ -12 +12 @@\\n-old\\n+new\"}]"; + } else if (reviewCommentList) { responseJson = includePreviousReviewComment - ? "[{\"id\":321,\"body\":\"old \"}]" + ? "[{\"id\":321,\"pull_request_review_id\":654," + + "\"body\":\"old \"}]" : "[]"; } else if (reviewList) { responseJson = includePreviousReviewComment diff --git a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatterTest.java b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatterTest.java index d64764ab1..1024dfd37 100644 --- a/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatterTest.java +++ b/java-ecosystem/services/pipeline-agent/src/test/java/org/rostilos/codecrow/pipelineagent/github/service/GitHubReviewFormatterTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.Test; import org.rostilos.codecrow.core.model.codeanalysis.IssueSeverity; import org.rostilos.codecrow.vcsclient.bitbucket.model.report.AnalysisSummary; +import org.rostilos.codecrow.vcsclient.github.actions.GetPullRequestDiffAction.PullRequestFilePatch; import java.util.List; import java.util.Map; @@ -25,9 +26,13 @@ void formatsAnchoredIssuesAsGitHubReviewComments() { "--- a/src/App.java\n+++ b/src/App.java\n@@ -42 +42 @@\n-old\n+new" ); - List> comments = formatter.formatComments(List.of(issue), MARKER); + GitHubReviewFormatter.ReviewPlan plan = formatter.planComments( + List.of(issue), MARKER, List.of(patch( + "src/App.java", "@@ -42 +42 @@\n-old\n+new"))); + List> comments = plan.inlineComments(); assertThat(comments).hasSize(1); + assertThat(plan.nonInlineFindings()).isEmpty(); assertThat(comments.get(0)) .containsEntry("path", "src/App.java") .containsEntry("line", 42) @@ -49,14 +54,17 @@ void skipsIssuesWithoutAConfidentLineAnchor() { AnalysisSummary.IssueSummary syntheticLineOne = issue( "src/App.java", 1, "Synthetic anchor", "Reason", null, null); - List> comments = formatter.formatComments( - List.of(noPath, noLine, syntheticLineOne), MARKER); + GitHubReviewFormatter.ReviewPlan plan = formatter.planComments( + List.of(noPath, noLine, syntheticLineOne), + MARKER, + List.of(patch("src/App.java", "@@ -1,10 +1,10 @@\n context"))); - assertThat(comments).isEmpty(); + assertThat(plan.inlineComments()).isEmpty(); + assertThat(plan.nonInlineFindings()).hasSize(3); } @Test - void keepsLineOneWhenTheIssueIncludesItsSourceSnippet() { + void keepsLineOneWithASourceSnippetOnlyWhenItIsInTheDiff() { AnalysisSummary.IssueSummary issue = new AnalysisSummary.IssueSummary( IssueSeverity.LOW, "CODE_QUALITY", @@ -71,7 +79,47 @@ void keepsLineOneWhenTheIssueIncludesItsSourceSnippet() { "package example;" ); - assertThat(formatter.formatComments(List.of(issue), MARKER)).hasSize(1); + GitHubReviewFormatter.ReviewPlan outsideDiff = formatter.planComments( + List.of(issue), + MARKER, + List.of(patch("src/App.java", "@@ -20 +20 @@\n-old\n+new"))); + GitHubReviewFormatter.ReviewPlan newFile = formatter.planComments( + List.of(issue), + MARKER, + List.of(patch("src/App.java", "@@ -0,0 +1,2 @@\n+package example;\n+class App {}"))); + + assertThat(outsideDiff.inlineComments()).isEmpty(); + assertThat(outsideDiff.nonInlineFindings()).singleElement() + .satisfies(finding -> assertThat(finding.reason()) + .contains("outside the current pull-request diff")); + assertThat(newFile.inlineComments()).hasSize(1); + assertThat(newFile.nonInlineFindings()).isEmpty(); + } + + @Test + void separatesOutOfDiffFindingsWithoutDroppingValidComments() { + AnalysisSummary.IssueSummary valid = issue( + "src/App.java", 42, "Valid anchor", "Reason", null, null); + AnalysisSummary.IssueSummary outside = issue( + "src/App.java", 67, "File-wide finding", "Reason", null, null); + + GitHubReviewFormatter.ReviewPlan plan = formatter.planComments( + List.of(valid, outside), + MARKER, + List.of(patch("src/App.java", "@@ -42 +42 @@\n-old\n+new"))); + + assertThat(plan.inlineComments()).singleElement() + .satisfies(comment -> assertThat(comment).containsEntry("line", 42)); + assertThat(plan.nonInlineFindings()).singleElement() + .satisfies(finding -> { + assertThat(finding.issue().getTitle()).isEqualTo("File-wide finding"); + assertThat(finding.reason()).contains("outside the current pull-request diff"); + }); + assertThat(formatter.formatNonInlineFindings(plan.nonInlineFindings())) + .contains("📍 Findings not posted inline (1)") + .contains("[File-wide finding](https://codecrow.example/issues/1)") + .contains("`src/App.java:67`") + .contains("outside the current pull-request diff"); } @Test @@ -79,13 +127,32 @@ void limitsTheReviewToTwentyInlineComments() { AnalysisSummary.IssueSummary issue = issue( "src/App.java", 10, "Title", "Reason", null, null); - assertThat(formatter.formatComments(java.util.Collections.nCopies(25, issue), MARKER)) - .hasSize(20); + GitHubReviewFormatter.ReviewPlan plan = formatter.planComments( + java.util.Collections.nCopies(25, issue), + MARKER, + List.of(patch("src/App.java", "@@ -10 +10 @@\n-old\n+new"))); + + assertThat(plan.inlineComments()).hasSize(20); + assertThat(plan.nonInlineFindings()).hasSize(5) + .allSatisfy(finding -> assertThat(finding.reason()).contains("limit of 20")); assertThat(formatter.formatReviewBody(20, MARKER)) .contains("**Actionable comments posted: 20**") .endsWith(MARKER); } + @Test + void keepsAllFindingsInTheSummaryWhenTheDiffCannotBeLoaded() { + AnalysisSummary.IssueSummary issue = issue( + "src/App.java", 10, "Title", "Reason", null, null); + + GitHubReviewFormatter.ReviewPlan plan = formatter.planWithoutDiff(List.of(issue)); + + assertThat(plan.inlineComments()).isEmpty(); + assertThat(plan.nonInlineFindings()).singleElement() + .satisfies(finding -> assertThat(finding.reason()) + .contains("diff could not be loaded")); + } + private AnalysisSummary.IssueSummary issue( String path, Integer line, @@ -107,4 +174,8 @@ private AnalysisSummary.IssueSummary issue( 1L ); } + + private PullRequestFilePatch patch(String path, String patch) { + return new PullRequestFilePatch(path, "", "modified", patch); + } } diff --git a/python-ecosystem/inference-orchestrator/integration/test_qa_documentation.py b/python-ecosystem/inference-orchestrator/integration/test_qa_documentation.py index 24a3bc5a4..af1353aa5 100644 --- a/python-ecosystem/inference-orchestrator/integration/test_qa_documentation.py +++ b/python-ecosystem/inference-orchestrator/integration/test_qa_documentation.py @@ -32,6 +32,30 @@ async def test_qa_doc_success(client, auth_headers): assert data.get("documentation") is not None or data.get("error") is not None +@pytest.mark.asyncio(loop_scope="function") +async def test_qa_doc_accepts_empty_jira_description(client, auth_headers): + """A Jira task without a description must not block diff-based QA generation.""" + mock_svc = AsyncMock() + mock_svc.generate = AsyncMock(return_value={ + "documentation": "## QA Documentation\nTest changes", + "documentation_needed": True, + }) + payload = _minimal_qa_payload() + payload["task_context"] = { + "task_key": "PROJ-123", + "task_summary": "Implement feature", + "description": None, + } + + with patch("api.routers.qa_documentation.QaDocumentationService", return_value=mock_svc): + resp = await client.post("/qa-documentation", json=payload, headers=auth_headers) + + assert resp.status_code == 200 + generation_args = mock_svc.generate.await_args.kwargs + assert generation_args["task_context"]["description"] == "" + assert generation_args["diff"] == payload["diff"] + + @pytest.mark.asyncio(loop_scope="function") async def test_qa_doc_validation_error(client, auth_headers): """Missing required fields → 422.""" diff --git a/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py b/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py index ea6c2c40f..cbfdd63b0 100644 --- a/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py +++ b/python-ecosystem/inference-orchestrator/src/api/routers/qa_documentation.py @@ -9,7 +9,7 @@ """ import logging from fastapi import APIRouter, Request -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from typing import Optional, Dict, Any, List, Literal from model.enrichment import PrEnrichmentDataDto @@ -71,6 +71,17 @@ class QaDocumentationRequest(BaseModel): oauth_secret: Optional[str] = None bearer_token: Optional[str] = None + @field_validator("task_context", mode="before") + @classmethod + def normalize_task_context_values(cls, value): + """Keep nullable provider fields compatible with the string prompt contract.""" + if value is None or not isinstance(value, dict): + return value + return { + key: "" if item is None else str(item) + for key, item in value.items() + } + class QaDocumentationResponse(BaseModel): """Response containing the generated QA documentation.""" diff --git a/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py b/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py index 92cd73f87..63a4bb077 100644 --- a/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py +++ b/python-ecosystem/inference-orchestrator/src/server/queue_consumer.py @@ -44,7 +44,7 @@ def __init__(self, review_service: ReviewService): int(self.consumer_heartbeat_seconds * 3), ) # Bound concurrent job processing to prevent memory pressure - max_concurrent = int(os.environ.get("MAX_CONCURRENT_REVIEWS", "4")) + max_concurrent = int(os.environ.get("MAX_CONCURRENT_REVIEWS", "20")) self._job_semaphore = asyncio.Semaphore(max_concurrent) self.heartbeat_seconds = max( 1.0, diff --git a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py index b82f7d644..6e384fe61 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/review_service.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/review_service.py @@ -50,7 +50,7 @@ class ReviewService: MAX_FIX_RETRIES = 2 # Maximum concurrent reviews (each spawns a JVM subprocess + LLM calls) - MAX_CONCURRENT_REVIEWS = int(os.environ.get("MAX_CONCURRENT_REVIEWS", "4")) + MAX_CONCURRENT_REVIEWS = int(os.environ.get("MAX_CONCURRENT_REVIEWS", "20")) # Hard timeout ceiling per review (seconds). Configurable via .env REVIEW_TIMEOUT_SECONDS = int(os.environ.get("REVIEW_TIMEOUT_SECONDS", "1500")) diff --git a/python-ecosystem/rag-pipeline/integration/conftest.py b/python-ecosystem/rag-pipeline/integration/conftest.py index 1a2d02893..a753d0f50 100644 --- a/python-ecosystem/rag-pipeline/integration/conftest.py +++ b/python-ecosystem/rag-pipeline/integration/conftest.py @@ -91,6 +91,7 @@ def rag_app(_mock_qdrant, _mock_embedding): assert_owned=MagicMock() ) mock_im.project_mutation.return_value = mutation_context + mock_im.pr_overlay_mutation.return_value = mutation_context mock_im.embed_model = _mock_embedding mock_im.qdrant_client = _mock_qdrant mock_im.splitter.split_documents_resilient.side_effect = ( diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py index 016fc87a1..b86625e47 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/api/routers/pr.py @@ -110,9 +110,10 @@ def index_pr_files(request: PRIndexRequest): completely and then replaces the previous points with rollback protection. """ index_manager = _get_index_manager() - mutation_context = index_manager.project_mutation( + mutation_context = index_manager.pr_overlay_mutation( request.workspace, request.project, + request.pr_number, "index-pr-overlay", ) mutation_lease = None @@ -826,9 +827,10 @@ def delete_pr_files( """Delete all indexed points for a specific PR.""" index_manager = _get_index_manager() try: - with index_manager.project_mutation( + with index_manager.pr_overlay_mutation( workspace, project, + pr_number, "delete-pr-overlay", ) as lease: collection_name = ( diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py index c990a869f..ee9358009 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py @@ -1040,6 +1040,26 @@ def project_mutation(self, workspace: str, project: str, operation: str): """Return the shared mutation boundary for auxiliary index endpoints.""" return self._mutation_coordinator.acquire(workspace, project, operation) + def pr_overlay_mutation( + self, + workspace: str, + project: str, + pr_number: int, + operation: str, + ): + """Serialize one PR overlay without blocking unrelated PRs. + + Index replacement and deletion for the same PR must not overlap, even + when the target generation changes between attempts. Different PR + numbers have disjoint point identities and can mutate concurrently. + """ + return self._mutation_coordinator.acquire( + workspace, + project, + operation, + publication_scope=f"pr-overlay:{pr_number}", + ) + def close(self) -> None: self._mutation_coordinator.close() diff --git a/python-ecosystem/rag-pipeline/tests/test_coordination.py b/python-ecosystem/rag-pipeline/tests/test_coordination.py index 0e52bfeb3..2d5603463 100644 --- a/python-ecosystem/rag-pipeline/tests/test_coordination.py +++ b/python-ecosystem/rag-pipeline/tests/test_coordination.py @@ -11,6 +11,7 @@ RedisPermitPool, ) from rag_pipeline.core.index_manager.collection_manager import CollectionManager +from rag_pipeline.core.index_manager.manager import RAGIndexManager def _coordinator(timeout=0): @@ -73,6 +74,42 @@ def test_branch_publication_scope_serializes_only_the_same_branch_head(): assert main != develop +def test_pr_overlay_scope_serializes_only_the_same_pr(): + coordinator = _coordinator() + + pr_41_index = coordinator._resource_key( + "workspace", "project", publication_scope="pr-overlay:41" + ) + pr_41_delete = coordinator._resource_key( + "workspace", "project", publication_scope="pr-overlay:41" + ) + pr_42_index = coordinator._resource_key( + "workspace", "project", publication_scope="pr-overlay:42" + ) + + assert pr_41_index == pr_41_delete + assert pr_41_index != pr_42_index + + +def test_index_manager_binds_overlay_mutations_to_pr_scope(): + coordinator = MagicMock() + manager = SimpleNamespace(_mutation_coordinator=coordinator) + lease = object() + coordinator.acquire.return_value = lease + + result = RAGIndexManager.pr_overlay_mutation( + manager, "workspace", "project", 42, "index-pr-overlay" + ) + + assert result is lease + coordinator.acquire.assert_called_once_with( + "workspace", + "project", + "index-pr-overlay", + publication_scope="pr-overlay:42", + ) + + def test_project_mutation_coordination_fails_closed_when_redis_is_unavailable(): coordinator = _coordinator() coordinator._client.set.side_effect = RuntimeError("redis unavailable") diff --git a/python-ecosystem/rag-pipeline/tests/test_router_pr.py b/python-ecosystem/rag-pipeline/tests/test_router_pr.py index 28eda8839..7a1276c2a 100644 --- a/python-ecosystem/rag-pipeline/tests/test_router_pr.py +++ b/python-ecosystem/rag-pipeline/tests/test_router_pr.py @@ -31,7 +31,7 @@ def _make_index_manager(): mutation_context.__enter__.return_value = SimpleNamespace( assert_owned=MagicMock() ) - im.project_mutation.return_value = mutation_context + im.pr_overlay_mutation.return_value = mutation_context im.index_representation_fingerprint = REPRESENTATION_FINGERPRINT im.pr_overlay_representation_fingerprint = ( OVERLAY_REPRESENTATION_FINGERPRINT @@ -103,6 +103,19 @@ def _capabilities(*plugin_ids, fingerprint="sha256:capabilities"): # ───────────────────────────────────────────────────────────── class TestIndexPRFiles: + @patch("rag_pipeline.api.routers.pr._get_index_manager") + def test_uses_pr_scoped_mutation_boundary(self, mock_get): + im = _make_index_manager() + mock_get.return_value = im + + from rag_pipeline.api.routers.pr import index_pr_files + + index_pr_files(_request([])) + + im.pr_overlay_mutation.assert_called_once_with( + "ws", "proj", 42, "index-pr-overlay" + ) + @patch("rag_pipeline.api.routers.pr._get_index_manager") def test_collection_target_without_base_revision_is_rejected(self, mock_get): im = _make_index_manager() @@ -801,6 +814,9 @@ def test_success(self, mock_get): result = delete_pr_files("ws", "proj", 42) assert result["status"] == "deleted" assert result["pr_number"] == 42 + im.pr_overlay_mutation.assert_called_once_with( + "ws", "proj", 42, "delete-pr-overlay" + ) selector = im.qdrant_client.delete.call_args.kwargs["points_selector"] coordinates = { From dbc72b60bb5c95c2efdd5f08e5d160798c2cfa3a Mon Sep 17 00:00:00 2001 From: rostislav Date: Mon, 10 Aug 2026 17:19:57 +0300 Subject: [PATCH 4/6] fix(review): make semantic dedup and MCP verification recall-safe Group only plausible duplicate candidates, preserve uncertain findings, and retain all merged locations. Bind MCP verification to the exact reviewed commit, finding ID, file, and source window while keeping model context token-efficient. --- .../service/IssueDeduplicationService.java | 184 +++- .../IssueDeduplicationServiceTest.java | 81 ++ .../rostilos/codecrow/mcp/McpStdioServer.java | 16 +- .../org/rostilos/codecrow/mcp/McpTools.java | 91 +- .../mcp/McpToolsSourceWindowTest.java | 67 ++ .../inference-orchestrator/src/README.MD | 47 +- .../src/model/output_schemas.py | 44 + .../review/orchestrator/inference_policy.py | 11 +- .../review/orchestrator/mcp_tool_executor.py | 225 ++++- .../review/orchestrator/orchestrator.py | 167 +++- .../review/orchestrator/reconciliation.py | 814 +++++++++++++++--- .../orchestrator/stage_3_aggregation.py | 295 ++++++- .../src/utils/prompts/constants_mcp.py | 31 +- .../src/utils/prompts/prompt_builder.py | 6 +- .../tests/test_inference_policy.py | 15 +- .../tests/test_mcp_tool_executor.py | 173 ++++ .../tests/test_orchestrator_helpers.py | 94 ++ .../tests/test_prompt_builder.py | 13 + .../tests/test_reconciliation.py | 53 +- .../tests/test_reconciliation_full.py | 219 ++++- .../tests/test_stage_3_full.py | 88 +- .../tests/test_stage_3_helpers.py | 163 +++- 22 files changed, 2599 insertions(+), 298 deletions(-) create mode 100644 java-ecosystem/mcp-servers/vcs-mcp/src/test/java/org/rostilos/codecrow/mcp/McpToolsSourceWindowTest.java diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java index fd530e295..9ae74079f 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java @@ -21,6 +21,9 @@ *

*

De-duplication identities

*
    + *
  1. Exact narrative identity — same normalized title and complete + * root-cause reason in the same file. Category, severity, and a stale line + * anchor do not make the same finding independent.
    2. *
  2. Issue fingerprint — exact match on * {@link org.rostilos.codecrow.core.util.tracking.IssueFingerprint} * (category + anchored line-content hash + normalized title).
    3. @@ -38,7 +41,8 @@ *
      *
    • The highest severity among the group
      • *
    • The best (longest valid) suggested fix diff
      • - *
    • The lowest line number (most specific location)
      • + *
    • The strongest concrete source anchor
      • + *
    • Additional concrete locations for exact narrative occurrences
      • *
    * *

    Thread safety

    @@ -105,6 +109,10 @@ public List deduplicateAtIngestion(List is int before = active.size(); + // Exact root narrative catches duplicated history/current output even + // when line anchors and classifications drifted before ingestion. + active = exactNarrativeDedup(active, filePath); + // Exact category-aware identity. active = fingerprintDedup(active, filePath); @@ -130,7 +138,127 @@ public List deduplicateAtIngestion(List is return result; } - // ── Tier 1: category-aware fingerprint ─────────────────────────────── + // ── Tier 1: complete root-narrative identity ───────────────────────── + + private List exactNarrativeDedup( + List issues, + String filePath + ) { + if (issues.size() < 2) { + return issues; + } + + Map byNarrative = new LinkedHashMap<>(); + int noIdentityOrdinal = 0; + int duplicateCount = 0; + for (CodeAnalysisIssue issue : issues) { + String title = normalizeNarrative(issue.getTitle()); + String reason = normalizeNarrative(rootReason(issue.getReason())); + // Short boilerplate (for example "same issue" / "test reason") is + // not a safe root identity. The inference tier handles ambiguous + // prose; this ingestion fallback requires a complete explanation. + if (title.isBlank() || reason.length() < 40) { + byNarrative.put("__no_narrative_" + noIdentityOrdinal++, issue); + continue; + } + + String identity = title + "\u0000" + reason; + CodeAnalysisIssue existing = byNarrative.get(identity); + if (existing == null) { + byNarrative.put(identity, issue); + continue; + } + + CodeAnalysisIssue winner = pickBest(existing, issue); + preserveAdditionalNarrativeLocation(winner, existing, issue); + byNarrative.put(identity, winner); + duplicateCount++; + } + + if (duplicateCount > 0) { + log.debug("Exact-narrative dedup: removed {} in {}", duplicateCount, filePath); + } + return new ArrayList<>(byNarrative.values()); + } + + private String normalizeNarrative(String value) { + if (value == null) { + return ""; + } + return value.strip().replaceAll("\\s+", " ").toLowerCase(Locale.ROOT); + } + + private String rootReason(String reason) { + if (reason == null || reason.isBlank()) { + return ""; + } + return Arrays.stream(reason.split("\\R")) + .filter(line -> !line.strip().toLowerCase(Locale.ROOT).startsWith("also affects:")) + .reduce((left, right) -> left + "\n" + right) + .orElse("") + .strip(); + } + + private void preserveAdditionalNarrativeLocation( + CodeAnalysisIssue winner, + CodeAnalysisIssue first, + CodeAnalysisIssue second + ) { + Set locations = new TreeSet<>(); + collectGeneratedLocations(first.getReason(), locations); + collectGeneratedLocations(second.getReason(), locations); + addConcreteSecondaryLocation(winner, first, locations); + addConcreteSecondaryLocation(winner, second, locations); + + String primary = issueLocation(winner); + locations.remove(primary); + String baseReason = rootReason(winner.getReason()); + winner.setReason(locations.isEmpty() + ? baseReason + : baseReason + "\n\nAlso affects: " + String.join(", ", locations)); + } + + private void collectGeneratedLocations(String reason, Set locations) { + if (reason == null) { + return; + } + for (String line : reason.split("\\R")) { + String stripped = line.strip(); + if (!stripped.toLowerCase(Locale.ROOT).startsWith("also affects:")) { + continue; + } + String value = stripped.substring(stripped.indexOf(':') + 1); + Arrays.stream(value.split(",")) + .map(String::strip) + .filter(item -> !item.isBlank()) + .forEach(locations::add); + } + } + + private void addConcreteSecondaryLocation( + CodeAnalysisIssue winner, + CodeAnalysisIssue candidate, + Set locations + ) { + if (candidate == winner || candidate.getLineNumber() == null + || candidate.getLineNumber() <= 1) { + // Line 1 is the legacy FILE-scope/stale-anchor fallback and is not + // safe to publish as an additional concrete occurrence. + return; + } + String location = issueLocation(candidate); + if (!location.isBlank()) { + locations.add(location); + } + } + + private String issueLocation(CodeAnalysisIssue issue) { + String file = issue.getFilePath() != null ? issue.getFilePath() : ""; + Integer line = issue.getLineNumber(); + return line != null && line > 0 ? file + ":" + line : file; + } + + // ── Tier 2: category-aware fingerprint ─────────────────────────────── /** * De-duplicate by issue fingerprint — catches issues at different lines @@ -214,38 +342,44 @@ private List contentFingerprintDedup(List /** * Pick the best representation of two issues that already have the same exact - * identity. Higher severity wins, then the longer valid diff, then the lower - * anchored line number. This method never decides whether two issues are equal. + * identity, then promote useful metadata from the other representation. This + * method never decides whether two issues are equal. */ private CodeAnalysisIssue pickBest(CodeAnalysisIssue a, CodeAnalysisIssue b) { - int sevA = severityRank(a.getSeverity()); - int sevB = severityRank(b.getSeverity()); - if (sevA != sevB) { - return sevA >= sevB ? promoteSeverity(a, b) : promoteSeverity(b, a); - } + CodeAnalysisIssue winner = representationRank(a) >= representationRank(b) ? a : b; + CodeAnalysisIssue loser = winner == a ? b : a; + promoteMergedMetadata(winner, loser); + return winner; + } - int diffLenA = validDiffLength(a.getSuggestedFixDiff()); - int diffLenB = validDiffLength(b.getSuggestedFixDiff()); - if (diffLenA != diffLenB) { - return diffLenA >= diffLenB ? a : b; + private long representationRank(CodeAnalysisIssue issue) { + long rank = 0; + if (issue.getCodeSnippet() != null && !issue.getCodeSnippet().isBlank()) { + rank += 1_000_000_000L; } - - int lineA = a.getLineNumber() != null ? a.getLineNumber() : Integer.MAX_VALUE; - int lineB = b.getLineNumber() != null ? b.getLineNumber() : Integer.MAX_VALUE; - return lineA <= lineB ? a : b; + if (issue.getLineNumber() != null && issue.getLineNumber() > 1) { + rank += 100_000_000L; + } else if (issue.getLineNumber() != null && issue.getLineNumber() > 0) { + rank += 10_000_000L; + } + rank += Math.min(validDiffLength(issue.getSuggestedFixDiff()), 1_000_000); + rank += Math.min(issue.getReason() != null ? issue.getReason().length() : 0, 100_000); + return rank; } - /** - * Return the winner but ensure it carries the highest severity from either issue. - */ - private CodeAnalysisIssue promoteSeverity(CodeAnalysisIssue winner, CodeAnalysisIssue loser) { - // Winner already has higher severity, but adopt loser's diff if winner lacks one - if (validDiffLength(winner.getSuggestedFixDiff()) == 0 - && validDiffLength(loser.getSuggestedFixDiff()) > 0) { + private void promoteMergedMetadata(CodeAnalysisIssue winner, CodeAnalysisIssue loser) { + if (severityRank(loser.getSeverity()) > severityRank(winner.getSeverity())) { + winner.setSeverity(loser.getSeverity()); + } + if (validDiffLength(loser.getSuggestedFixDiff()) + > validDiffLength(winner.getSuggestedFixDiff())) { winner.setSuggestedFixDiff(loser.getSuggestedFixDiff()); winner.setSuggestedFixDescription(loser.getSuggestedFixDescription()); + } else if ((winner.getSuggestedFixDescription() == null + || winner.getSuggestedFixDescription().isBlank()) + && loser.getSuggestedFixDescription() != null) { + winner.setSuggestedFixDescription(loser.getSuggestedFixDescription()); } - return winner; } private int severityRank(IssueSeverity severity) { diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/IssueDeduplicationServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/IssueDeduplicationServiceTest.java index ac30e0b1c..3bdbea4bd 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/IssueDeduplicationServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/IssueDeduplicationServiceTest.java @@ -299,6 +299,87 @@ void shouldMergeSameAnchoredIssueWhenCategoryDrifts() { } } + @Nested + @DisplayName("Exact narrative safety net") + class ExactNarrativeDedup { + + @Test + @DisplayName("should collapse recreated history despite anchor and category drift") + void shouldCollapseRecreatedHistoryWithCurrentAnchor() { + CodeAnalysisIssue staleHistory = createIssue( + "src/RetryService.java", 1, IssueCategory.CODE_QUALITY, + IssueSeverity.MEDIUM, "Unbounded retry loop can exhaust workers"); + staleHistory.setReason("The retry loop has no terminal attempt limit."); + staleHistory.setCodeSnippet("public class RetryService {"); + staleHistory.setIssueFingerprint(null); + staleHistory.setContentFingerprint(null); + + CodeAnalysisIssue current = createIssue( + "src/RetryService.java", 1233, IssueCategory.BUG_RISK, + IssueSeverity.HIGH, "Unbounded retry loop can exhaust workers"); + current.setReason("The retry loop has no terminal attempt limit."); + current.setCodeSnippet("while (shouldRetry(response)) {"); + current.setIssueFingerprint(null); + current.setContentFingerprint(null); + + List result = service.deduplicateAtIngestion( + new ArrayList<>(List.of(staleHistory, current))); + + assertThat(result).hasSize(1); + assertThat(result.get(0).getLineNumber()).isEqualTo(1233); + assertThat(result.get(0).getCodeSnippet()) + .isEqualTo("while (shouldRetry(response)) {"); + assertThat(result.get(0).getSeverity()).isEqualTo(IssueSeverity.HIGH); + assertThat(result.get(0).getReason()).doesNotContain("RetryService.java:1"); + } + + @Test + @DisplayName("should retain every concrete occurrence location") + void shouldRetainAdditionalOccurrenceLocation() { + CodeAnalysisIssue first = createIssue( + "src/Policy.java", 10, IssueCategory.SECURITY, + IssueSeverity.HIGH, "Authorization check is missing"); + CodeAnalysisIssue second = createIssue( + "src/Policy.java", 20, IssueCategory.BUG_RISK, + IssueSeverity.MEDIUM, "Authorization check is missing"); + first.setReason("The update executes without checking workspace ownership."); + second.setReason("The update executes without checking workspace ownership."); + first.setIssueFingerprint(null); + first.setContentFingerprint(null); + second.setIssueFingerprint(null); + second.setContentFingerprint(null); + + List result = service.deduplicateAtIngestion( + new ArrayList<>(List.of(first, second))); + + assertThat(result).hasSize(1); + assertThat(result.get(0).getReason()) + .contains("Also affects: src/Policy.java:20"); + } + + @Test + @DisplayName("should keep independent root causes at one anchor") + void shouldKeepIndependentRootCausesAtOneAnchor() { + CodeAnalysisIssue authorization = createIssue( + "src/Policy.java", 25, IssueCategory.BUG_RISK, + IssueSeverity.HIGH, "Request processing defect"); + CodeAnalysisIssue transaction = createIssue( + "src/Policy.java", 25, IssueCategory.BUG_RISK, + IssueSeverity.HIGH, "Request processing defect"); + authorization.setReason("Workspace ownership is never checked."); + transaction.setReason("The database transaction commits too early."); + authorization.setIssueFingerprint(null); + authorization.setContentFingerprint(null); + transaction.setIssueFingerprint(null); + transaction.setContentFingerprint(null); + + List result = service.deduplicateAtIngestion( + new ArrayList<>(List.of(authorization, transaction))); + + assertThat(result).hasSize(2); + } + } + // ── Resolved issues ────────────────────────────────────────────────── @Nested diff --git a/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpStdioServer.java b/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpStdioServer.java index c04a8b34e..764fc085f 100644 --- a/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpStdioServer.java +++ b/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpStdioServer.java @@ -621,12 +621,26 @@ private static List defineTools() { "filePath": { "type": "string", "description": "File Path." + }, + "startLine": { + "type": "integer", + "minimum": 1, + "description": "Optional first source line for an anchor-centred verification window." + }, + "endLine": { + "type": "integer", + "minimum": 1, + "description": "Optional last source line for the verification window. The server returns at most 401 lines." } }, "required": ["workspace", "repoSlug", "branch", "filePath"] } """; - tools.add(new Tool("getBranchFileContent", "Get full file content from specified branch or commit.", getBranchFileContentSchema)); + tools.add(new Tool( + "getBranchFileContent", + "Get file content or an optional whole-line range from a branch or commit.", + getBranchFileContentSchema + )); String getRootDirectorySchema = """ diff --git a/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpTools.java b/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpTools.java index 9372a7d61..afd49a2cc 100644 --- a/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpTools.java +++ b/java-ecosystem/mcp-servers/vcs-mcp/src/main/java/org/rostilos/codecrow/mcp/McpTools.java @@ -1,6 +1,7 @@ package org.rostilos.codecrow.mcp; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -15,6 +16,7 @@ import org.slf4j.LoggerFactory; public class McpTools { + private static final int MAX_SOURCE_WINDOW_LINES = 401; private final VcsMcpClientFactory vcsMcpClientFactory; private VcsMcpClient vcsClient = null; private final LargeContentFilter largeContentFilter; @@ -166,7 +168,9 @@ public Object execute(String toolName, Map arguments) throws IOE (String) arguments.get("workspace"), (String) arguments.get("repoSlug"), (String) arguments.get("branch"), - (String) arguments.get("filePath") + (String) arguments.get("filePath"), + integerArgument(arguments.get("startLine")), + integerArgument(arguments.get("endLine")) ); case "getRootDirectory": return getRootDirectory( @@ -319,16 +323,97 @@ public Map getPullRequestCommits(String workspace, String repoSl } public Map getBranchFileContent(String workspace, String repoSlug, String branch, String filePath) { + return getBranchFileContent(workspace, repoSlug, branch, filePath, null, null); + } + + public Map getBranchFileContent( + String workspace, + String repoSlug, + String branch, + String filePath, + Integer startLine, + Integer endLine + ) { try { String fileContent = getVcsClient(true).getBranchFileContent(workspace, repoSlug, branch, filePath); - // Filter large file content to reduce token usage + if (startLine != null && startLine > 0) { + return sourceWindow(fileContent, startLine, endLine); + } + + // Exploratory callers without an anchor retain the existing large-file + // safeguard. Review verification supplies an issue line and receives + // a bounded source window instead of this generic placeholder. String filteredContent = largeContentFilter.filterFileContent(fileContent, filePath); - return Map.of("fileContent", filteredContent); + boolean completeFile = filteredContent != null + && !filteredContent.contains(LargeContentFilter.FILTERED_PLACEHOLDER); + int totalLines = lineCount(fileContent); + Map response = new LinkedHashMap<>(); + response.put("fileContent", filteredContent != null ? filteredContent : ""); + response.put("startLine", completeFile && totalLines > 0 ? 1 : 0); + response.put("endLine", completeFile ? totalLines : 0); + response.put("totalLines", totalLines); + response.put("completeFile", completeFile); + return response; } catch (IOException e) { return Map.of("error", "Failed to get branch file content: " + e.getMessage()); } } + static Map sourceWindow( + String content, + int requestedStartLine, + Integer requestedEndLine + ) { + if (content == null) { + return Map.of("error", "File content is unavailable"); + } + String[] lines = content.split("\\R", -1); + int totalLines = lines.length; + int startLine = Math.max(1, requestedStartLine); + if (startLine > totalLines) { + return Map.of( + "error", "Requested source line is outside the file", + "totalLines", totalLines + ); + } + int requestedEnd = requestedEndLine != null && requestedEndLine >= startLine + ? requestedEndLine + : startLine; + int endLine = Math.min( + totalLines, + Math.min(requestedEnd, startLine + MAX_SOURCE_WINDOW_LINES - 1) + ); + String fileContent = String.join( + "\n", + java.util.Arrays.copyOfRange(lines, startLine - 1, endLine) + ); + Map response = new LinkedHashMap<>(); + response.put("fileContent", fileContent); + response.put("startLine", startLine); + response.put("endLine", endLine); + response.put("totalLines", totalLines); + response.put("completeFile", startLine == 1 && endLine == totalLines); + return response; + } + + private static int lineCount(String content) { + return content == null ? 0 : content.split("\\R", -1).length; + } + + private static Integer integerArgument(Object value) { + if (value instanceof Number number) { + return number.intValue(); + } + if (value instanceof String text) { + try { + return Integer.valueOf(text.trim()); + } catch (NumberFormatException ignored) { + return null; + } + } + return null; + } + public Map getRootDirectory(String workspace, String projectKey, String branch) { try { String rootDirectory = getVcsClient().getRootDirectory(workspace, projectKey, branch); diff --git a/java-ecosystem/mcp-servers/vcs-mcp/src/test/java/org/rostilos/codecrow/mcp/McpToolsSourceWindowTest.java b/java-ecosystem/mcp-servers/vcs-mcp/src/test/java/org/rostilos/codecrow/mcp/McpToolsSourceWindowTest.java new file mode 100644 index 000000000..61083c12b --- /dev/null +++ b/java-ecosystem/mcp-servers/vcs-mcp/src/test/java/org/rostilos/codecrow/mcp/McpToolsSourceWindowTest.java @@ -0,0 +1,67 @@ +package org.rostilos.codecrow.mcp; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +class McpToolsSourceWindowTest { + + @Test + void returnsAnchorCentredLinesWithCoverageMetadata() { + String content = numberedLines(1_000); + + Map result = McpTools.sourceWindow(content, 420, 580); + + assertThat(result.get("startLine")).isEqualTo(420); + assertThat(result.get("endLine")).isEqualTo(580); + assertThat(result.get("totalLines")).isEqualTo(1_000); + assertThat(result.get("completeFile")).isEqualTo(false); + assertThat((String) result.get("fileContent")) + .startsWith("line-420\n") + .endsWith("line-580"); + } + + @Test + void boundsAnOverwideRequestAtWholeLineBoundaries() { + Map result = McpTools.sourceWindow( + numberedLines(1_000), + 1, + 1_000 + ); + + assertThat(result.get("startLine")).isEqualTo(1); + assertThat(result.get("endLine")).isEqualTo(401); + assertThat(result.get("completeFile")).isEqualTo(false); + assertThat(((String) result.get("fileContent")).lines()).hasSize(401); + } + + @Test + void marksACompleteSmallFile() { + Map result = McpTools.sourceWindow( + "first\nsecond\nthird", + 1, + 3 + ); + + assertThat(result.get("fileContent")).isEqualTo("first\nsecond\nthird"); + assertThat(result.get("completeFile")).isEqualTo(true); + } + + @Test + void rejectsAnAnchorOutsideTheFile() { + Map result = McpTools.sourceWindow("one\ntwo", 3, 5); + + assertThat(result).containsKey("error"); + assertThat(result.get("totalLines")).isEqualTo(2); + } + + private String numberedLines(int count) { + return IntStream.rangeClosed(1, count) + .mapToObj(index -> "line-" + index) + .collect(Collectors.joining("\n")); + } +} diff --git a/python-ecosystem/inference-orchestrator/src/README.MD b/python-ecosystem/inference-orchestrator/src/README.MD index 455fe76ec..0547d2ce0 100644 --- a/python-ecosystem/inference-orchestrator/src/README.MD +++ b/python-ecosystem/inference-orchestrator/src/README.MD @@ -508,9 +508,17 @@ finding. This applies to `FILE` scope as well as line/block/function scope, and missing, stale, or out-of-hunk anchors are removed before verifier calls. For unused-import-like claims, the gate also rejects the claim when the named symbol is visibly referenced elsewhere. The same publication gate runs again after Stage 2, and rejected -Stage 2 candidates are removed from Stage 3 report context. LLM tool verification -remains a secondary check, and its file-content cache is request-local for -concurrent reviews. +Stage 2 candidates are removed from Stage 3 report context. Optional Stage 3 MCP +verification remains a secondary check. Every active finding receives a stable +execution-local Verification ID, and every source read is host-pinned to the +immutable reviewed commit. For a concrete line, the host asks the VCS MCP server +for an anchor-centred source window instead of sending an unrelated full large +file or accepting a size-filter placeholder. A dismissal is accepted only when +the response metadata binds the same Verification ID, path, revision, and a +window covering the primary and every consolidated location. Empty content, +tool errors, filter placeholders, missing locations, wrong revisions, +comments-only evidence, or ambiguity keeps the finding. Fresh findings without +database IDs use the same object-identity path. RAG latency safeguards keep slow semantic search from blocking analysis while preserving deterministic context: @@ -530,17 +538,28 @@ Per-batch LLM reranking is disabled by default (`LLM_RERANK_ENABLED=false`) to avoid one extra model call per batch. The same retrieved chunks are preserved for the review model; set `LLM_RERANK_ENABLED=true` to opt into listwise reranking. -Final issue deduplication is also conservative and local by default -(`REVIEW_LLM_DEDUP_ENABLED=false`). It removes a candidate only when normalized -file, category, exact line or exact source snippet, and closely matching -root-cause text agree. Similar wording in different files or at different source -anchors is retained. Set `REVIEW_LLM_DEDUP_ENABLED=true` only when the deployment -explicitly accepts extra review-model calls and model-directed suppression. -The Java persistence boundary does not broaden that identity: it merges only -exact category-aware or category-agnostic anchored fingerprints. Sharing a file, -line, category, or FILE scope cannot by itself suppress another finding, and an -issue whose fingerprint lacks an actual line hash or exact snippet anchor passes -through unchanged. These checks add no model calls. +Final issue deduplication begins with exact local root identities and uses grouped +semantic dedup by default (`REVIEW_LLM_DEDUP_ENABLED=true`). The host sends only +ambiguous candidate components; unrelated singletons consume no dedup tokens. +The model returns explicit duplicate groups, and only `HIGH`-confidence decisions +inside one host-selected component can merge findings. Omitted, uncertain, +malformed, or failed decisions retain all ambiguous findings. Category or +severity drift does not split a proven root cause. Cross-file consolidation is +allowed only for one shared causal defect and retains every secondary `file:line` +in the finding's reason and `relatedLocations`. + +Candidate components are packed near `REVIEW_DEDUP_BATCH_CHAR_BUDGET=48000` +without field-level clipping; an oversized component is sent intact by itself. +`REVIEW_DEDUP_MAX_PARALLEL=4` limits candidate batches within one review and does +not serialize concurrently admitted reviews. Set `REVIEW_LLM_DEDUP_ENABLED=false` +to use exact local merging only. + +The Java persistence boundary adds an exact same-file substantive-title/reason +safety net before its exact category-aware and category-agnostic anchored +fingerprints. It retains the strongest source anchor, highest severity, best +valid fix, and additional concrete locations. Short boilerplate, sharing only a +file/line/category/FILE scope, and placeholder fingerprints without an actual +anchor cannot suppress another finding. These Java checks add no model calls. Full-pipeline prompt dry runs forward the normal Stage 0–3 progress events and persist a compact `review_evidence_completed` record with terminal hunk counts, diff --git a/python-ecosystem/inference-orchestrator/src/model/output_schemas.py b/python-ecosystem/inference-orchestrator/src/model/output_schemas.py index 441a0089f..d94a4a49b 100644 --- a/python-ecosystem/inference-orchestrator/src/model/output_schemas.py +++ b/python-ecosystem/inference-orchestrator/src/model/output_schemas.py @@ -88,6 +88,15 @@ def normalize_scope(cls, v) -> str: "Leave empty for a generic defect proved directly by changed source." ), ) + relatedLocations: List[str] = Field( + default_factory=list, + description=( + "Additional repository-relative file:line locations consolidated " + "under the same root-cause finding. The primary file and line remain " + "in file/line; this list prevents semantic deduplication from losing " + "other affected locations." + ), + ) @field_validator('codeSnippet', mode='before') @classmethod @@ -127,6 +136,41 @@ class DeduplicatedIssueList(BaseModel): ) +class SemanticDuplicateGroup(BaseModel): + """One explicit, high-confidence semantic duplicate decision.""" + + keeper_index: int = Field( + description="Batch-local index of the best representative to keep." + ) + duplicate_indices: List[int] = Field( + default_factory=list, + description=( + "Batch-local indices describing the same root cause as keeper_index." + ), + ) + confidence: str = Field( + description=( + "HIGH, MEDIUM, or LOW confidence. Only HIGH decisions are eligible " + "for host-side merging." + ), + ) + rationale: str = Field( + default="", + description="Short root-cause identity explanation, not chain-of-thought.", + ) + + +class SemanticDeduplicationDecision(BaseModel): + """Recall-safe semantic dedup output. + + Findings omitted from duplicate_groups are retained. This is intentionally + safer than a kept-index allowlist, where a malformed or incomplete response + can accidentally delete unrelated findings. + """ + + duplicate_groups: List[SemanticDuplicateGroup] = Field(default_factory=list) + + class AskOutput(BaseModel): """Schema for ask command output.""" answer: str = Field(description="Well-formatted markdown answer to the user's question") diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/inference_policy.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/inference_policy.py index 29ef0b5ca..5109f00a3 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/inference_policy.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/inference_policy.py @@ -40,7 +40,7 @@ def _env_int(name: str, default: int) -> int: OUTPUT_CAP_MODEL_KWARG = os.environ.get("REVIEW_OUTPUT_CAP_MODEL_KWARG", "").strip() FAST_CHECK_ENABLED = _env_bool("REVIEW_FAST_CHECK_ENABLED", True) STAGE_2_ENABLED = _env_bool("REVIEW_STAGE_2_ENABLED", True) -LLM_DEDUP_ENABLED = _env_bool("REVIEW_LLM_DEDUP_ENABLED", False) +LLM_DEDUP_ENABLED = _env_bool("REVIEW_LLM_DEDUP_ENABLED", True) FAST_CHECK_MAX_FILES = _env_int("REVIEW_FAST_CHECK_MAX_FILES", 4) FAST_CHECK_MAX_CHANGED_LINES = _env_int("REVIEW_FAST_CHECK_MAX_CHANGED_LINES", 800) @@ -188,12 +188,9 @@ def should_use_llm_dedup( profile: ReviewInferenceProfile, issue_count: int, ) -> bool: - """LLM dedup is an explicit cost/quality opt-in, never a default stage.""" - return ( - LLM_DEDUP_ENABLED - and issue_count > 1 - and not should_use_fast_dedup(profile, issue_count) - ) + """Enable grouped semantic dedup; singleton candidate sets skip internally.""" + del profile + return LLM_DEDUP_ENABLED and issue_count > 1 def _count_review_files( diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/mcp_tool_executor.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/mcp_tool_executor.py index a39fd3b28..a4800927e 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/mcp_tool_executor.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/mcp_tool_executor.py @@ -5,11 +5,20 @@ Stage 3 (issue verification): getBranchFileContent, getPullRequestComments — max 5 calls total """ import asyncio +import json import logging +import re from typing import Any, Dict, List, Optional, Set logger = logging.getLogger(__name__) +_REVIEW_SOURCE_CONTEXT_LINES = 80 +_FILTERED_CONTENT_MARKER = "[CodeCrow Filter:" +_MCP_ERROR_PREFIXES = ("Error executing tool:", "Tool call failed:") +_RELATED_LOCATIONS_RE = re.compile( + r"(?im)^\s*(?:[*_]{1,2})?also affects\s*:(?:[*_]{1,2})?\s*(.+)$" +) + class McpToolExecutor: """ @@ -31,7 +40,14 @@ class McpToolExecutor: }, } - def __init__(self, mcp_client, request, stage: str): + def __init__( + self, + mcp_client, + request, + stage: str, + review_revision: Optional[str] = None, + verification_issues: Optional[Dict[str, Any]] = None, + ): if stage not in self.STAGE_CONFIG: raise ValueError(f"Unknown stage '{stage}'. Valid: {list(self.STAGE_CONFIG)}") @@ -43,6 +59,8 @@ def __init__(self, mcp_client, request, stage: str): self.max_calls: int = config["max_calls"] self.call_count: int = 0 self.call_log: List[Dict[str, Any]] = [] + self.review_revision = str(review_revision or "").strip() + self.verification_issues = dict(verification_issues or {}) self._lock = asyncio.Lock() # ------------------------------------------------------------------ @@ -66,8 +84,35 @@ async def execute_tool(self, tool_name: str, arguments: Dict[str, Any]) -> str: # Pre-fill workspace/repo from request context so the LLM doesn't # have to guess these values. + arguments = dict(arguments or {}) arguments.setdefault("workspace", self.request.projectVcsWorkspace) arguments.setdefault("repoSlug", self.request.projectVcsRepoSlug) + if ( + self.stage == "stage_3" + and tool_name == "getBranchFileContent" + and self.review_revision + ): + # Post-review evidence must come from the exact reviewed revision. + # Never let a model accidentally verify new PR code against target. + arguments["branch"] = self.review_revision + verification_id = str( + arguments.get("verificationId") or "" + ).strip() + source_line = self._verification_line_for_path( + verification_id, + str(arguments.get("filePath") or ""), + ) + if source_line > 0: + # Request an anchor-centred source window. This avoids replacing + # large files with a generic size placeholder or sending an + # unrelated full file merely to verify one concrete finding. + arguments["startLine"] = max( + 1, + source_line - _REVIEW_SOURCE_CONTEXT_LINES, + ) + arguments["endLine"] = ( + source_line + _REVIEW_SOURCE_CONTEXT_LINES + ) logger.info( f"[MCP {self.stage}] Calling {tool_name} " @@ -76,15 +121,48 @@ async def execute_tool(self, tool_name: str, arguments: Dict[str, Any]) -> str: try: result = await self.client.session.call_tool(tool_name, arguments) - self.call_log.append( - {"tool": tool_name, "args": arguments, "success": True} - ) # Extract text content from MCP result - if hasattr(result, "content") and result.content: - return "\n".join( - block.text for block in result.content if hasattr(block, "text") + if hasattr(result, "content"): + text = "\n".join( + block.text + for block in (result.content or []) + if hasattr(block, "text") ) - return str(result) + else: + text = str(result) + evidence = self._file_evidence_metadata(tool_name, text) + tool_reported_error = bool( + getattr(result, "isError", False) + or getattr(result, "is_error", False) + ) + if tool_reported_error: + evidence = self._file_evidence_metadata(tool_name, "") + if ( + self.stage == "stage_3" + and tool_name == "getBranchFileContent" + and evidence.get("evidence_valid") is True + and evidence.get("evidence_structured") is not True + and arguments.get("startLine") + ): + # Legacy adapters may return raw source rather than the VCS MCP + # metadata envelope. Since Stage 3 explicitly requested a + # window, bind that raw response to the requested range instead + # of incorrectly treating it as proof for the entire file. + evidence["evidence_complete_file"] = False + evidence["evidence_start_line"] = arguments["startLine"] + evidence["evidence_end_line"] = arguments.get( + "endLine", + arguments["startLine"], + ) + log_entry = { + "tool": tool_name, + "args": dict(arguments), + "success": not tool_reported_error, + "result_chars": len(text.strip()), + **evidence, + } + self.call_log.append(log_entry) + return text except Exception as e: logger.error(f"[MCP {self.stage}] Tool call failed: {e}") self.call_log.append( @@ -101,20 +179,40 @@ def get_tool_definitions(self) -> List[Dict[str, Any]]: "type": "function", "function": { "name": "getBranchFileContent", - "description": "Read a file's content from the target branch.", + "description": ( + "Read repository file content. For Stage 3, provide the " + "finding Verification ID; the host pins the reviewed " + "revision and requests an anchor-centred source window." + if self.stage == "stage_3" + else "Read a file's content from the target branch." + ), "parameters": { "type": "object", "properties": { - "branch": { - "type": "string", - "description": "Branch name (e.g. 'main', 'develop')" - }, + **({ + "branch": { + "type": "string", + "description": "Target branch name (for example main).", + }, + } if self.stage != "stage_3" else {}), "filePath": { "type": "string", "description": "Path to the file in the repository" }, + **({ + "verificationId": { + "type": "string", + "description": ( + "Verification ID from the Stage 3 finding record." + ), + }, + } if self.stage == "stage_3" else {}), }, - "required": ["branch", "filePath"], + "required": ( + ["filePath", "verificationId"] + if self.stage == "stage_3" + else ["branch", "filePath"] + ), }, }, }) @@ -153,3 +251,102 @@ def summary(self) -> str: f"{self.call_count}/{self.max_calls} calls used, " f"{len(self.call_log)} logged" ) + + @staticmethod + def _normalized_path(value: Any) -> str: + return str(value or "").strip().replace("\\", "/").lstrip("/") + + @staticmethod + def _location_parts(value: Any) -> tuple[str, int]: + normalized = McpToolExecutor._normalized_path(value) + path, separator, possible_line = normalized.rpartition(":") + if separator and possible_line.isdigit(): + return path, int(possible_line) + return normalized, 0 + + def _verification_line_for_path( + self, + verification_id: str, + file_path: str, + ) -> int: + issue = self.verification_issues.get(verification_id) + if issue is None: + return 0 + requested_path = self._normalized_path(file_path) + issue_path = self._normalized_path(getattr(issue, "file", "")) + if requested_path == issue_path: + try: + return max(0, int(getattr(issue, "line", 0) or 0)) + except (TypeError, ValueError): + return 0 + for location in self._related_locations(issue): + path, line = self._location_parts(location) + if path == requested_path: + return line + return 0 + + @staticmethod + def _related_locations(issue: Any) -> List[str]: + values = list(getattr(issue, "relatedLocations", None) or []) + reason = str(getattr(issue, "reason", "") or "") + for match in _RELATED_LOCATIONS_RE.finditer(reason): + values.extend(match.group(1).split(",")) + return sorted({ + str(value).strip() + for value in values + if str(value).strip() + }) + + @staticmethod + def _file_evidence_metadata( + tool_name: str, + text: str, + ) -> Dict[str, Any]: + if tool_name != "getBranchFileContent": + return {} + + stripped = str(text or "").strip() + metadata: Dict[str, Any] = { + "evidence_valid": False, + "evidence_structured": False, + "evidence_complete_file": False, + "evidence_start_line": 0, + "evidence_end_line": 0, + } + if ( + not stripped + or _FILTERED_CONTENT_MARKER in stripped + or stripped.startswith(_MCP_ERROR_PREFIXES) + ): + return metadata + + try: + payload = json.loads(stripped) + except (json.JSONDecodeError, TypeError): + # Some MCP adapters return raw source text instead of a JSON map. + metadata["evidence_valid"] = True + metadata["evidence_complete_file"] = True + return metadata + + if not isinstance(payload, dict) or payload.get("error"): + return metadata + metadata["evidence_structured"] = True + file_content = payload.get("fileContent") + if not isinstance(file_content, str) or not file_content.strip(): + return metadata + if _FILTERED_CONTENT_MARKER in file_content: + return metadata + + def integer(name: str) -> int: + try: + return max(0, int(payload.get(name) or 0)) + except (TypeError, ValueError): + return 0 + + metadata.update({ + "evidence_valid": True, + "evidence_complete_file": payload.get("completeFile") is True, + "evidence_start_line": integer("startLine"), + "evidence_end_line": integer("endLine"), + }) + return metadata diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py index bf31881f2..888736fb9 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/orchestrator.py @@ -1297,20 +1297,16 @@ async def orchestrate_review( # === FINAL DEDUP: after ALL issue-finding stages (1 + 1.5 + 2) === # Historical resolutions are lifecycle updates, not competing - # findings. Keep them out of both dedup implementations, which are - # intentionally content-based and could otherwise discard the update. + # findings. Active historical identities do participate so duplicate + # history and fresh recreations can be consolidated. The merge keeps + # one persisted identity and emits explicit close updates for any + # superseded historical IDs. active_issues, resolved_lifecycle_issues = _partition_issue_lifecycle( file_issues ) - fresh_active_issues, protected_active_issues = ( - _partition_protected_active_issues( - active_issues, - protected_open_issue_ids, - ) - ) - pre_dedup_count = len(fresh_active_issues) - if not fresh_active_issues: - deduplicated_fresh_issues = [] + pre_dedup_count = len(active_issues) + if not active_issues: + deduplicated_active_issues = [] elif should_use_llm_dedup( inference_profile, pre_dedup_count, @@ -1319,13 +1315,13 @@ async def orchestrate_review( self.event_callback, "final_dedup_started", ( - "Final dedup: opt-in semantic LLM dedup for " + "Final dedup: grouped recall-safe semantic dedup for " f"{pre_dedup_count} issue(s)" ), ) - deduplicated_fresh_issues = await deduplicate_final_issues_llm( + deduplicated_active_issues = await deduplicate_final_issues_llm( with_stage_output_cap(self.llm, "dedup", inference_profile), - fresh_active_issues, + active_issues, ) else: fast_dedup = should_use_fast_dedup( @@ -1349,37 +1345,50 @@ async def orchestrate_review( f"{pre_dedup_count} issue(s)" ), ) - deduplicated_fresh_issues = deduplicate_final_issues( - fresh_active_issues + deduplicated_active_issues = deduplicate_final_issues( + active_issues ) - before_final_dedup = list(fresh_active_issues) + before_final_dedup = list(active_issues) candidate_ledger.reject_removed( before_final_dedup, - deduplicated_fresh_issues, + deduplicated_active_issues, gate="deduplication", code="final_duplicate", ) - before_history_suppression = list(deduplicated_fresh_issues) - deduplicated_fresh_issues = _suppress_duplicates_of_protected_history( - deduplicated_fresh_issues, - protected_active_issues, - ) - candidate_ledger.reject_removed( - before_history_suppression, - deduplicated_fresh_issues, - gate="deduplication", - code="duplicate_of_open_history", - ) - if len(deduplicated_fresh_issues) != pre_dedup_count: + + retained_object_ids = { + id(issue) for issue in deduplicated_active_issues + } + consolidated_history: List[CodeReviewIssue] = [] + for removed_issue in before_final_dedup: + if id(removed_issue) in retained_object_ids: + continue + removed_id = str(getattr(removed_issue, "id", "") or "").strip() + if removed_id not in protected_open_issue_ids: + continue + resolved_copy = _resolved_historical_copy( + removed_issue, + protected_open_issue_ids, + ( + "Closed because final root-cause deduplication " + "consolidated this duplicate into the retained finding." + ), + ) + if resolved_copy is not None: + consolidated_history.append(resolved_copy) + + if len(deduplicated_active_issues) != pre_dedup_count: logger.info( - "Final dedup before Stage 3: %d → %d fresh active issues", + "Final dedup before Stage 3: %d → %d active root findings " + "(%d historical duplicate(s) closed)", pre_dedup_count, - len(deduplicated_fresh_issues), + len(deduplicated_active_issues), + len(consolidated_history), ) file_issues = ( - deduplicated_fresh_issues - + protected_active_issues + deduplicated_active_issues + resolved_lifecycle_issues + + consolidated_history ) # Stage 3 receives the structured Stage 2 result separately from the @@ -1414,25 +1423,34 @@ async def orchestrate_review( pr_evidence_ledger.task_implementation_evidence_payload(task_key) ) dismissed_ids = set(stage_3_result.get("dismissed_issue_ids", [])) + dismissed_object_ids = { + int(value) + for value in stage_3_result.get( + "dismissed_issue_object_ids", + [], + ) + } # A dismissed historical OPEN issue is a lifecycle update, not an # omission. Return it as resolved so the client can close the stored # record; only genuinely fresh candidates are removed outright. - if dismissed_ids: + if dismissed_ids or dismissed_object_ids: before_stage_3_dismissals = list(file_issues) file_issues, resolved_count, dropped_count = ( _apply_stage_3_dismissals( file_issues, dismissed_ids, protected_open_issue_ids, + dismissed_object_ids=dismissed_object_ids, ) ) logger.info( "Stage 3 dismissed %d fresh issue(s) and resolved %d " - "historical OPEN issue(s) (IDs: %s)", + "historical OPEN issue(s) after evidence validation " + "(verification keys: %s)", dropped_count, resolved_count, - dismissed_ids, + stage_3_result.get("dismissed_issue_keys", []), ) candidate_ledger.reject_removed( before_stage_3_dismissals, @@ -1773,6 +1791,30 @@ def _normalized_issue_resolution(issue: CodeReviewIssue) -> Optional[str]: return None +def _resolved_historical_copy( + issue: CodeReviewIssue, + previous_open_ids: set[str], + explanation: str, +) -> Optional[CodeReviewIssue]: + """Create a lifecycle close update without re-publishing rejected provenance. + + Reconciled historical objects can be bound to a generated-candidate ledger + record. Dedup rejects the superseded candidate object, while this unbound copy + is returned solely so persistence can close the old database identity. + """ + if hasattr(issue, "model_copy"): + resolved = issue.model_copy(deep=True) + else: + resolved = CodeReviewIssue(**issue.model_dump()) + if not _resolve_historical_candidate( + resolved, + previous_open_ids, + explanation, + ): + return None + return resolved + + def _partition_protected_active_issues( active_issues: List[CodeReviewIssue], protected_ids: set[str], @@ -1817,18 +1859,38 @@ def _deduplicate_cross_batch_issues_preserving_lifecycle( issues: List[CodeReviewIssue], protected_ids: Optional[set[str]] = None, ) -> List[CodeReviewIssue]: - """Deduplicate fresh Stage 1 findings without losing historical identity.""" + """Deduplicate Stage 1 findings while retaining lifecycle close updates. + + Active history participates in exact merging so its persisted identity can + absorb a fresh, better source anchor. If two persisted OPEN records collapse + into one root finding, the superseded ID is returned as an explicit resolved + update instead of disappearing. + """ active, resolved = _partition_issue_lifecycle(issues) fresh, protected = _partition_protected_active_issues( active, protected_ids or set(), ) - deduplicated_fresh = deduplicate_cross_batch_issues(fresh) - deduplicated_fresh = _suppress_duplicates_of_protected_history( - deduplicated_fresh, - protected, - ) - return deduplicated_fresh + protected + resolved + # Preserve the established publication order (fresh, protected, resolved) + # while still allowing exact merging across the fresh/history boundary. + ordered_active = fresh + protected + deduplicated_active = deduplicate_cross_batch_issues(ordered_active) + retained_object_ids = {id(issue) for issue in deduplicated_active} + consolidated_history: List[CodeReviewIssue] = [] + for issue in ordered_active: + if id(issue) in retained_object_ids: + continue + resolved_copy = _resolved_historical_copy( + issue, + protected_ids or set(), + ( + "Closed because exact root-cause deduplication consolidated " + "this duplicate into the retained finding." + ), + ) + if resolved_copy is not None: + consolidated_history.append(resolved_copy) + return deduplicated_active + resolved + consolidated_history def _serialize_issue_for_client(issue: CodeReviewIssue) -> Dict[str, Any]: @@ -1858,8 +1920,15 @@ def _apply_stage_3_dismissals( issues: List[CodeReviewIssue], dismissed_ids: set[str], previous_open_ids: set[str], + *, + dismissed_object_ids: Optional[set[int]] = None, ) -> tuple[List[CodeReviewIssue], int, int]: - """Close dismissed OPEN history and drop only fresh false positives.""" + """Close verified OPEN history and drop only verified fresh false positives. + + Object identities are preferred because Stage 3 verification IDs cover fresh + findings that do not have a database ID and avoid touching resolved lifecycle + records that happen to share a persisted ID. + """ normalized_dismissed_ids = { str(issue_id).strip() for issue_id in dismissed_ids @@ -1868,10 +1937,16 @@ def _apply_stage_3_dismissals( retained: List[CodeReviewIssue] = [] resolved_count = 0 dropped_count = 0 + use_object_identity = bool(dismissed_object_ids) for issue in issues: issue_id = str(getattr(issue, "id", "") or "").strip() - if issue_id not in normalized_dismissed_ids: + is_dismissed = ( + id(issue) in (dismissed_object_ids or set()) + if use_object_identity + else issue_id in normalized_dismissed_ids + ) + if not is_dismissed: retained.append(issue) continue diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/reconciliation.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/reconciliation.py index 129fbea4b..cf6a8a38c 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/reconciliation.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/reconciliation.py @@ -4,11 +4,16 @@ import logging import difflib import asyncio +import json import os +import re from collections import defaultdict -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Sequence -from model.output_schemas import CodeReviewIssue, DeduplicatedIssueList +from model.output_schemas import ( + CodeReviewIssue, + SemanticDeduplicationDecision, +) from service.review.candidate_ledger import CandidateEvidenceLedger from utils.llm_response import extract_llm_response_text from service.review.orchestrator.json_utils import parse_llm_response, supports_structured_output @@ -159,6 +164,263 @@ def _issue_payload(issue: Any) -> Dict[str, Any]: return vars(issue) if hasattr(issue, "__dict__") else {} +_TEXT_TOKEN_RE = re.compile(r"[A-Za-z0-9_$]+") +_ROOT_STOP_WORDS = { + "a", "an", "and", "are", "as", "at", "be", "because", "by", "can", + "could", "for", "from", "has", "have", "if", "in", "into", "is", "it", + "may", "of", "on", "or", "that", "the", "their", "this", "to", "was", + "were", "will", "with", "would", +} +_SEVERITY_RANK = { + "CRITICAL": 5, + "HIGH": 4, + "MEDIUM": 3, + "LOW": 2, + "INFO": 1, +} + + +def _normalized_file(data: Dict[str, Any]) -> str: + value = str(data.get("file") or data.get("filePath") or "").strip() + normalized = value.replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + return normalized.lstrip("/") + + +def _split_reason_locations(value: Any) -> tuple[str, set[str]]: + """Separate root-cause prose from host-generated affected locations.""" + body: List[str] = [] + locations: set[str] = set() + for line in str(value or "").splitlines(): + stripped = line.strip() + if stripped.casefold().startswith("also affects:"): + raw_locations = stripped.split(":", 1)[1] + locations.update( + item.strip() + for item in raw_locations.split(",") + if item.strip() + ) + else: + body.append(line) + return "\n".join(body).strip(), locations + + +def _normalized_text(value: Any) -> str: + body, _ = _split_reason_locations(value) + return " ".join(token.casefold() for token in _TEXT_TOKEN_RE.findall(body)) + + +def _meaningful_tokens(value: Any) -> set[str]: + return { + token + for token in _normalized_text(value).split() + if token not in _ROOT_STOP_WORDS and len(token) > 1 + } + + +def _text_similarity(left: Any, right: Any) -> float: + normalized_left = _normalized_text(left) + normalized_right = _normalized_text(right) + if not normalized_left or not normalized_right: + return 0.0 + if normalized_left == normalized_right: + return 1.0 + return difflib.SequenceMatcher( + None, + normalized_left, + normalized_right, + ).ratio() + + +def _token_overlap(left: Any, right: Any) -> tuple[int, float, float]: + left_tokens = _meaningful_tokens(left) + right_tokens = _meaningful_tokens(right) + if not left_tokens or not right_tokens: + return 0, 0.0, 0.0 + overlap = len(left_tokens & right_tokens) + containment = overlap / min(len(left_tokens), len(right_tokens)) + union = len(left_tokens | right_tokens) + return overlap, containment, overlap / union if union else 0.0 + + +def _line_number(data: Dict[str, Any]) -> int: + try: + return int(data.get("line") or data.get("lineNumber") or 0) + except (TypeError, ValueError): + return 0 + + +def _issue_id(issue: Any) -> str: + return str(_issue_payload(issue).get("id") or "").strip() + + +def _issue_location(issue: Any) -> str: + data = _issue_payload(issue) + file_path = _normalized_file(data) + line = _line_number(data) + return f"{file_path}:{line}" if line > 0 else file_path + + +def _history_identity_rank(issue: Any) -> tuple[int, int, str]: + issue_id = _issue_id(issue) + if not issue_id: + return 0, 0, "" + try: + # Prefer the oldest numeric identity when historical duplicates already + # exist, preserving the longest-lived review/comment lineage. + return 1, -int(issue_id), issue_id + except ValueError: + return 1, 0, issue_id + + +def _representation_rank(issue: Any) -> tuple[int, int, int, int]: + data = _issue_payload(issue) + line = _line_number(data) + snippet = str(data.get("codeSnippet") or data.get("code_snippet") or "").strip() + reason, _ = _split_reason_locations(data.get("reason") or "") + fix = str(data.get("suggestedFixDescription") or "").strip() + return ( + 1 if snippet else 0, + 1 if line > 1 else 0, + len(reason), + len(fix), + ) + + +def _set_issue_field(issue: Any, name: str, value: Any) -> None: + if hasattr(issue, name): + setattr(issue, name, value) + + +def _merge_duplicate_issues( + left: Any, + right: Any, + *, + prefer_left: bool = False, +) -> Any: + """Merge two proven root-cause duplicates without losing locations. + + A persisted identity wins over a fresh object, while a fresh exact anchor + refreshes a stale historical location. The function mutates and returns one + input object so candidate-ledger provenance remains attached. + """ + left_history = _history_identity_rank(left) + right_history = _history_identity_rank(right) + if left_history != right_history: + canonical = left if left_history > right_history else right + elif prefer_left: + canonical = left + else: + canonical = ( + left + if _representation_rank(left) >= _representation_rank(right) + else right + ) + other = right if canonical is left else left + + canonical_data = _issue_payload(canonical) + other_data = _issue_payload(other) + left_location_before_merge = _issue_location(left) + right_location_before_merge = _issue_location(right) + same_file = _normalized_file(canonical_data) == _normalized_file(other_data) + + # A current fresh anchor is stronger than a carried historical hint. For two + # fresh findings, prefer the representation with the stronger concrete anchor. + if same_file: + if bool(_issue_id(canonical)) != bool(_issue_id(other)): + anchor_source = other if not _issue_id(other) else canonical + elif prefer_left: + anchor_source = canonical + else: + anchor_source = ( + canonical + if _representation_rank(canonical) >= _representation_rank(other) + else other + ) + anchor_data = _issue_payload(anchor_source) + for field in ("file", "line", "scope", "codeSnippet"): + value = anchor_data.get(field) + if value not in (None, ""): + _set_issue_field(canonical, field, value) + + canonical_reason, canonical_locations = _split_reason_locations( + canonical_data.get("reason") + ) + other_reason, other_locations = _split_reason_locations(other_data.get("reason")) + detail_source = canonical if len(canonical_reason) >= len(other_reason) else other + base_reason = canonical_reason if detail_source is canonical else other_reason + + locations = set(canonical_locations) | set(other_locations) + for value in canonical_data.get("relatedLocations") or []: + if str(value).strip(): + locations.add(str(value).strip()) + for value in other_data.get("relatedLocations") or []: + if str(value).strip(): + locations.add(str(value).strip()) + + primary_location = _issue_location(canonical) + if not same_file: + locations.update({ + left_location_before_merge, + right_location_before_merge, + }) + elif ( + (not _issue_id(left) and not _issue_id(right)) + or ( + _issue_id(left) + and _issue_id(right) + and _issue_id(left) != _issue_id(right) + ) + ): + # Distinct occurrences of one root cause remain visible after merging. + # A history/current pair is excluded because its line difference usually + # represents a refreshed anchor for the same persisted occurrence. + locations.update({ + left_location_before_merge, + right_location_before_merge, + }) + locations.discard("") + locations.discard(primary_location) + sorted_locations = sorted(locations) + merged_reason = base_reason + if sorted_locations: + merged_reason = ( + f"{base_reason}\n\nAlso affects: {', '.join(sorted_locations)}" + ).strip() + _set_issue_field(canonical, "reason", merged_reason) + _set_issue_field(canonical, "relatedLocations", sorted_locations) + + for field in ("title", "suggestedFixDescription", "suggestedFixDiff"): + current = canonical_data.get(field) + alternative = other_data.get(field) + best = max( + (current, alternative), + key=lambda value: len(str(value)) if value else 0, + ) + if best and best != current: + _set_issue_field(canonical, field, best) + + left_severity = str(_issue_payload(left).get("severity") or "").upper() + right_severity = str(_issue_payload(right).get("severity") or "").upper() + highest = max( + (left_severity, right_severity), + key=lambda value: _SEVERITY_RANK.get(value, 0), + ) + if highest: + _set_issue_field(canonical, "severity", highest) + + evidence_refs = sorted({ + str(value).strip() + for issue in (left, right) + for value in (_issue_payload(issue).get("evidenceRefs") or []) + if str(value).strip() + }) + if evidence_refs: + _set_issue_field(canonical, "evidenceRefs", evidence_refs) + return canonical + + def _normalized_anchor(value: Any) -> str: return " ".join(str(value or "").split()) @@ -191,7 +453,7 @@ def _issues_share_exact_plugin_proof( left: Dict[str, Any], right: Dict[str, Any], ) -> bool: - """Use plugin proof identity as a prose-independent root-cause key.""" + """Use shared plugin proof to nominate a semantic duplicate candidate.""" left_kind = str( left.get("claimKind") or left.get("claim_kind") or "" ).strip() @@ -222,45 +484,178 @@ def issues_are_conservative_duplicates( left: Any, right: Any, ) -> bool: - """Require location, category, anchor, and root-cause agreement. + """Return true only for deterministic, effectively exact root identities. - Similar prose alone is not an issue identity. Repeated guard/validation - defects in different files or at different lines remain separate findings. + Category and severity drift do not make a real duplicate independent. At + this tier, however, prose similarity alone is insufficient: uncertain pairs + are delegated to the grouped semantic pass and retained if that pass fails. """ left_data = _issue_payload(left) right_data = _issue_payload(right) - left_file = str( - left_data.get("file") or left_data.get("filePath") or "" - ).replace("\\", "/") - right_file = str( - right_data.get("file") or right_data.get("filePath") or "" - ).replace("\\", "/") + left_file = _normalized_file(left_data) + right_file = _normalized_file(right_data) if not left_file or left_file != right_file: return False - # Selected plugins supply stable proof identities. An exact match ties both - # candidates to the same framework/language relationship even when model - # prose, category, or the chosen line anchor differs across batches. - if _issues_share_exact_plugin_proof(left_data, right_data): + left_title = _normalized_text(left_data.get("title") or "") + right_title = _normalized_text(right_data.get("title") or "") + left_reason = _normalized_text( + left_data.get("reason") or left_data.get("description") or "" + ) + right_reason = _normalized_text( + right_data.get("reason") or right_data.get("description") or "" + ) + exact_narrative = bool( + left_title + and left_title == right_title + and left_reason + and left_reason == right_reason + ) + if exact_narrative: return True - left_category = str(left_data.get("category") or "").upper() - right_category = str(right_data.get("category") or "").upper() - if not left_category or left_category != right_category: - return False if not _issues_share_exact_anchor(left_data, right_data): return False + if left_reason and left_reason == right_reason: + return True + if ( + left_reason + and right_reason + and _text_similarity(left_reason, right_reason) >= 0.88 + ): + return True + return bool( + left_title + and left_title == right_title + and _text_similarity(left_reason, right_reason) >= 0.94 + ) + + +def issues_are_semantic_dedup_candidates(left: Any, right: Any) -> bool: + """Select plausible duplicate pairs for semantic comparison. + + This is a candidate-recall operation, not a deletion decision. It is broad + enough to include anchor/category wording drift, while cross-file grouping + requires a shared non-generic title and substantial technical-token overlap. + """ + if issues_are_conservative_duplicates(left, right): + return True + + left_data = _issue_payload(left) + right_data = _issue_payload(right) + left_file = _normalized_file(left_data) + right_file = _normalized_file(right_data) + if not left_file or not right_file: + return False + + # A stable plugin fact is strong enough to justify semantic comparison, but + # not deterministic deletion: one structural relationship can support more + # than one genuinely distinct defect claim. + if _issues_share_exact_plugin_proof(left_data, right_data): + return True + left_title = left_data.get("title") or "" + right_title = right_data.get("title") or "" left_reason = left_data.get("reason") or left_data.get("description") or "" - right_reason = ( - right_data.get("reason") or right_data.get("description") or "" + right_reason = right_data.get("reason") or right_data.get("description") or "" + title_similarity = _text_similarity(left_title, right_title) + reason_similarity = _text_similarity(left_reason, right_reason) + title_overlap, title_containment, title_jaccard = _token_overlap( + left_title, + right_title, ) - return is_semantically_similar( - str(left_reason), - str(right_reason), - threshold=0.82, + reason_overlap, reason_containment, reason_jaccard = _token_overlap( + left_reason, + right_reason, ) + if left_file != right_file: + exact_title = bool( + _normalized_text(left_title) + and _normalized_text(left_title) == _normalized_text(right_title) + ) + # Cross-file occurrences are candidates only for a distinctive shared + # root signature. Generic repeated warnings remain independent. + return bool( + exact_title + and len(_meaningful_tokens(left_title)) >= 4 + and reason_overlap >= 4 + and reason_containment >= 0.50 + and reason_jaccard >= 0.25 + ) + + left_line = _line_number(left_data) + right_line = _line_number(right_data) + near_line = bool( + left_line > 0 + and right_line > 0 + and abs(left_line - right_line) <= 3 + ) + exact_anchor = _issues_share_exact_anchor(left_data, right_data) + exact_title = bool( + _normalized_text(left_title) + and _normalized_text(left_title) == _normalized_text(right_title) + ) + + if exact_title and ( + reason_similarity >= 0.50 + or (reason_overlap >= 3 and reason_containment >= 0.45) + ): + return True + + anchor_related = exact_anchor or near_line + title_related = bool( + title_similarity >= 0.48 + or (title_overlap >= 3 and title_containment >= 0.55) + or title_jaccard >= 0.42 + ) + reason_related = bool( + reason_similarity >= 0.46 + or (reason_overlap >= 4 and reason_containment >= 0.50) + or reason_jaccard >= 0.36 + ) + if title_related and ( + reason_similarity >= 0.70 + or (reason_overlap >= 5 and reason_containment >= 0.65) + ): + # Strong same-file narrative identity can drift to a nearby helper or + # call site. Send it for semantic judgment without deleting locally. + return True + return anchor_related and title_related and reason_related + + +def _semantic_candidate_groups( + issues: Sequence[CodeReviewIssue], +) -> List[List[CodeReviewIssue]]: + """Build connected candidate components; singleton findings cost no tokens.""" + count = len(issues) + parents = list(range(count)) + + def find(index: int) -> int: + while parents[index] != index: + parents[index] = parents[parents[index]] + index = parents[index] + return index + + def union(left_index: int, right_index: int) -> None: + left_root = find(left_index) + right_root = find(right_index) + if left_root != right_root: + parents[right_root] = left_root + + for left_index in range(count): + for right_index in range(left_index + 1, count): + if issues_are_semantic_dedup_candidates( + issues[left_index], + issues[right_index], + ): + union(left_index, right_index) + + groups: Dict[int, List[CodeReviewIssue]] = defaultdict(list) + for index, issue in enumerate(issues): + groups[find(index)].append(issue) + return [group for group in groups.values() if len(group) > 1] + def deduplicate_issues(issues: List[Any]) -> List[dict]: """Deduplicate issues by fingerprint, keeping most recent version. @@ -369,28 +764,33 @@ def deduplicate_final_issues(issues: List[CodeReviewIssue]) -> List[CodeReviewIs Final deduplication pass after ALL issue-finding stages complete (Stage 1, Reconciliation, Verification, Stage 2 cross-file). - Conservative deterministic dedup requires the same normalized file, - category, exact line or exact current-source snippet, and closely matching - root-cause text. Similar prose by itself never suppresses another finding. + The deterministic tier merges only effectively exact root identities. It + preserves the best anchor, highest severity, historical identity, and all + additional affected locations. Ambiguous semantic pairs remain untouched + for the grouped LLM pass. """ if not issues: return [] deduped: List[CodeReviewIssue] = [] for issue in issues: - if any( - issues_are_conservative_duplicates(issue, existing) - for existing in deduped - ): - data = _issue_payload(issue) - logger.info( - "Final deterministic dedup: suppressed anchored duplicate at " - "%s:%s", - data.get("file", ""), - data.get("line", ""), - ) + duplicate_index = next(( + index + for index, existing in enumerate(deduped) + if issues_are_conservative_duplicates(issue, existing) + ), None) + if duplicate_index is None: + deduped.append(issue) continue - deduped.append(issue) + existing = deduped[duplicate_index] + merged = _merge_duplicate_issues(existing, issue) + deduped[duplicate_index] = merged + data = _issue_payload(issue) + logger.info( + "Final deterministic dedup: merged exact root identity at %s:%s", + data.get("file", ""), + data.get("line", ""), + ) original = len(issues) final = len(deduped) @@ -405,22 +805,27 @@ def deduplicate_final_issues(issues: List[CodeReviewIssue]) -> List[CodeReviewIs _DEDUP_BATCH_SIZE = 50 _DEDUP_MAX_PARALLEL = max(1, _env_int("REVIEW_DEDUP_MAX_PARALLEL", 4)) +_DEDUP_BATCH_CHAR_BUDGET = max( + 8_000, + _env_int("REVIEW_DEDUP_BATCH_CHAR_BUDGET", 48_000), +) _DEDUP_SYSTEM_PROMPT = ( - "You are a code review deduplication assistant. You will receive a list of " - "code-review issues (each with an index, file, line, severity, category, and " - "reason). Your task is to identify **semantic duplicates** — issues that " - "describe the same underlying problem even if they use different wording, " - "slightly different line numbers in the same file, or were found by different " - "analysis stages.\n\n" + "You are a code-review root-cause deduplication assistant. You receive only " + "host-selected candidate groups; findings outside these groups are never sent " + "and are automatically retained. Identify only HIGH-confidence duplicate " + "occurrences that describe one actionable root cause.\n\n" "Rules:\n" - "1. Two issues are duplicates if they point to the SAME root cause in the " - "SAME file (small line-number differences are OK).\n" - "2. When you find duplicates, KEEP the one with the most detailed/useful " - "reason text and DROP the rest.\n" - "3. Issues in DIFFERENT files are NEVER duplicates of each other.\n" - "4. Return ONLY the 0-based indices of the issues you decide to KEEP.\n" - "5. If there are no duplicates at all, return every index." + "1. Compare findings only inside the same candidate_group.\n" + "2. Duplicate means the same causal defect, not merely the same rule, pattern, " + "category, or suggested fix. Independent occurrences must remain separate.\n" + "3. Category, severity, stage, and small anchor differences do not make the " + "same root cause independent.\n" + "4. Cross-file findings may be duplicates only when one shared defect causes " + "the reported failures; repeated independent defects in different files are not.\n" + "5. Select the most current, concrete, and useful representative as keeper.\n" + "6. Emit a duplicate group only at HIGH confidence. Omit uncertain pairs; " + "omitted findings are retained. Never return a kept-index allowlist." ) @@ -443,6 +848,97 @@ def _format_issues_for_prompt(issues: List[CodeReviewIssue]) -> str: return "\n".join(lines) +def _semantic_issue_payload( + issue: CodeReviewIssue, + index: int, + group_id: str, +) -> Dict[str, Any]: + data = _issue_payload(issue) + reason, generated_locations = _split_reason_locations( + data.get("reason") or data.get("description") or "" + ) + related_locations = sorted({ + str(value).strip() + for value in ( + list(data.get("relatedLocations") or []) + + list(generated_locations) + ) + if str(value).strip() + }) + return { + "index": index, + "candidate_group": group_id, + "existing_issue_id": str(data.get("id") or ""), + "file": _normalized_file(data), + "line": _line_number(data), + "severity": str(data.get("severity") or ""), + "category": str(data.get("category") or ""), + "title": str(data.get("title") or ""), + # Candidate grouping keeps the request small enough to preserve complete + # root-cause prose. No field-level character clipping is performed here. + "reason": reason, + "suggested_fix": str(data.get("suggestedFixDescription") or ""), + "exact_source_anchor": str( + data.get("codeSnippet") or data.get("code_snippet") or "" + ), + "related_locations": related_locations, + } + + +def _format_semantic_batch( + issues: Sequence[CodeReviewIssue], + group_by_index: Dict[int, str], +) -> str: + return json.dumps( + [ + _semantic_issue_payload(issue, index, group_by_index[index]) + for index, issue in enumerate(issues) + ], + ensure_ascii=False, + separators=(",", ":"), + ) + + +def _build_semantic_dedup_batches( + groups: Sequence[Sequence[CodeReviewIssue]], +) -> List[tuple[List[CodeReviewIssue], Dict[int, str]]]: + """Pack whole candidate groups by rendered size without clipping content.""" + batches: List[tuple[List[CodeReviewIssue], Dict[int, str]]] = [] + current_groups: List[Sequence[CodeReviewIssue]] = [] + current_chars = 0 + + def group_size(group: Sequence[CodeReviewIssue], group_index: int) -> int: + mapping = {index: f"candidate_{group_index}" for index in range(len(group))} + return len(_format_semantic_batch(group, mapping)) + + def flush() -> None: + nonlocal current_groups, current_chars + if not current_groups: + return + issues: List[CodeReviewIssue] = [] + mapping: Dict[int, str] = {} + for local_group_index, group in enumerate(current_groups): + group_id = f"candidate_{local_group_index}" + for issue in group: + mapping[len(issues)] = group_id + issues.append(issue) + batches.append((issues, mapping)) + current_groups = [] + current_chars = 0 + + for group_index, group in enumerate(groups): + rendered_chars = group_size(group, group_index) + if current_groups and current_chars + rendered_chars > _DEDUP_BATCH_CHAR_BUDGET: + flush() + current_groups.append(group) + current_chars += rendered_chars + # An unusually detailed group is sent alone with its evidence intact. + if rendered_chars > _DEDUP_BATCH_CHAR_BUDGET: + flush() + flush() + return batches + + def _build_batches(issues: List[CodeReviewIssue], max_batch_size: int = _DEDUP_BATCH_SIZE, ) -> List[List[CodeReviewIssue]]: @@ -477,45 +973,111 @@ def _build_batches(issues: List[CodeReviewIssue], async def _dedup_batch_with_llm( llm, batch: List[CodeReviewIssue], + group_by_index: Optional[Dict[int, str]] = None, ) -> List[CodeReviewIssue]: - """Send one batch to the LLM and return the kept issues.""" - issues_text = _format_issues_for_prompt(batch) + """Merge only validated high-confidence duplicate groups from one batch.""" + groups = group_by_index or {index: "candidate_0" for index in range(len(batch))} + issues_text = _format_semantic_batch(batch, groups) prompt = ( f"{_DEDUP_SYSTEM_PROMPT}\n\n" - f"Here are the issues to deduplicate:\n\n{issues_text}\n\n" - "Return the kept_indices list." + f"Candidate findings JSON:\n{issues_text}\n\n" + "Return duplicate_groups only." ) try: if supports_structured_output(llm): - structured_llm = llm.with_structured_output(DeduplicatedIssueList) - result: DeduplicatedIssueList = await structured_llm.ainvoke(prompt) + structured_llm = llm.with_structured_output( + SemanticDeduplicationDecision + ) + result: SemanticDeduplicationDecision = await structured_llm.ainvoke( + prompt + ) else: logger.info("Structured output skipped for LLM dedup batch; using prompt JSON parsing") response = await llm.ainvoke(prompt) result = await parse_llm_response( extract_llm_response_text(response), - DeduplicatedIssueList, + SemanticDeduplicationDecision, llm, ) - kept_indices = set(result.kept_indices) - # Sanity-check: indices must be within range - valid = {i for i in kept_indices if 0 <= i < len(batch)} - if not valid: - logger.warning( - "LLM dedup returned no valid indices — keeping all issues in batch" - ) - return batch + removed_indices: set[int] = set() + replacements: Dict[int, CodeReviewIssue] = {} + accepted_groups = 0 + for decision in result.duplicate_groups: + if str(decision.confidence or "").strip().upper() != "HIGH": + continue + keeper_index = decision.keeper_index + duplicate_indices = list(dict.fromkeys(decision.duplicate_indices)) + all_indices = [keeper_index, *duplicate_indices] + if ( + keeper_index < 0 + or keeper_index >= len(batch) + or not duplicate_indices + or any(index < 0 or index >= len(batch) for index in all_indices) + or keeper_index in duplicate_indices + or any(index in removed_indices for index in all_indices) + or any( + index in replacements and index != keeper_index + for index in duplicate_indices + ) + or len({groups.get(index) for index in all_indices}) != 1 + ): + logger.warning( + "LLM dedup rejected malformed or cross-group decision: %s", + decision.model_dump(), + ) + continue + keeper = replacements.get(keeper_index, batch[keeper_index]) + if not all( + issues_are_semantic_dedup_candidates( + keeper, + batch[duplicate_index], + ) + for duplicate_index in duplicate_indices + ): + logger.warning( + "LLM dedup rejected decision without host candidate evidence: %s", + decision.model_dump(), + ) + continue + for duplicate_index in duplicate_indices: + keeper = _merge_duplicate_issues( + keeper, + batch[duplicate_index], + prefer_left=True, + ) + removed_indices.add(duplicate_index) + replacements[keeper_index] = keeper + accepted_groups += 1 - kept = [batch[i] for i in sorted(valid)] - dropped = len(batch) - len(kept) - if dropped: - logger.info(f"LLM dedup batch: kept {len(kept)}/{len(batch)} issues (dropped {dropped})") + if not removed_indices: + return batch + kept: List[CodeReviewIssue] = [] + emitted_replacements: set[int] = set() + for index, issue in enumerate(batch): + if index in removed_indices: + continue + replacement = replacements.get(index, issue) + replacement_id = id(replacement) + if replacement_id in emitted_replacements: + continue + emitted_replacements.add(replacement_id) + kept.append(replacement) + logger.info( + "LLM semantic dedup accepted %d group(s): %d → %d candidate issues", + accepted_groups, + len(batch), + len(kept), + ) return kept except Exception as exc: - logger.warning(f"LLM dedup batch failed ({exc}); falling back to algorithmic dedup") + logger.warning( + "LLM dedup batch failed (%s); retaining ambiguous findings after " + "exact deterministic dedup", + exc, + ) return deduplicate_final_issues(batch) @@ -523,52 +1085,86 @@ async def deduplicate_final_issues_llm( llm, issues: List[CodeReviewIssue], ) -> List[CodeReviewIssue]: - """Primary LLM-driven deduplication. + """Recall-safe semantic dedup over host-selected candidate components. - 1. Groups issues by filepath. - 2. Packs filepath-groups into batches of ≤ 50 issues. - 3. Sends each batch to the LLM to identify semantic duplicates. - 4. Returns the union of kept issues from all batches. - - Falls back to ``deduplicate_final_issues`` (algorithmic) for any batch - where the LLM call fails. + Exact identities merge locally first. Only ambiguous components with two or + more plausible duplicates consume model tokens, and a failed/incomplete LLM + decision retains every ambiguous finding. """ if not issues: return [] - if len(issues) <= 1: - return issues + exact_deduped = deduplicate_final_issues(issues) + if len(exact_deduped) <= 1: + return exact_deduped + + candidate_groups = _semantic_candidate_groups(exact_deduped) + if not candidate_groups: + logger.info( + "LLM semantic dedup skipped: %d findings produced no ambiguous " + "candidate group", + len(exact_deduped), + ) + return exact_deduped - batches = _build_batches(issues, max_batch_size=_DEDUP_BATCH_SIZE) + batches = _build_semantic_dedup_batches(candidate_groups) + rendered_chars = sum( + len(_format_semantic_batch(batch, mapping)) + for batch, mapping in batches + ) + candidate_count = sum(len(group) for group in candidate_groups) logger.info( - f"LLM dedup: {len(issues)} issues split into {len(batches)} batch(es) " - f"(sizes: {[len(b) for b in batches]}, concurrency={_DEDUP_MAX_PARALLEL})" + "LLM semantic dedup: %d/%d findings in %d candidate group(s), " + "%d batch(es), input≈%d tokens, concurrency=%d", + candidate_count, + len(exact_deduped), + len(candidate_groups), + len(batches), + rendered_chars // 4, + _DEDUP_MAX_PARALLEL, ) semaphore = asyncio.Semaphore(_DEDUP_MAX_PARALLEL) batch_results: Dict[int, List[CodeReviewIssue]] = {} - async def _run_batch(batch_idx: int, batch: List[CodeReviewIssue]) -> tuple[int, List[CodeReviewIssue]]: + async def _run_batch( + batch_idx: int, + batch: List[CodeReviewIssue], + mapping: Dict[int, str], + ) -> tuple[int, List[CodeReviewIssue]]: async with semaphore: logger.info( f"LLM dedup: processing batch {batch_idx + 1}/{len(batches)} " f"({len(batch)} issues)" ) - kept = await _dedup_batch_with_llm(llm, batch) + kept = await _dedup_batch_with_llm(llm, batch, mapping) return batch_idx, kept tasks = [ - asyncio.create_task(_run_batch(batch_idx, batch)) - for batch_idx, batch in enumerate(batches) + asyncio.create_task(_run_batch(batch_idx, batch, mapping)) + for batch_idx, (batch, mapping) in enumerate(batches) ] for completed_task in asyncio.as_completed(tasks): batch_idx, kept = await completed_task batch_results[batch_idx] = kept - kept_issues: List[CodeReviewIssue] = [] - for batch_idx in range(len(batches)): - kept_issues.extend(batch_results.get(batch_idx, [])) + candidate_object_ids = { + id(issue) + for group in candidate_groups + for issue in group + } + retained_candidate_ids = { + id(issue) + for kept in batch_results.values() + for issue in kept + } + removed_candidate_ids = candidate_object_ids - retained_candidate_ids + kept_issues = [ + issue + for issue in exact_deduped + if id(issue) not in removed_candidate_ids + ] original = len(issues) final = len(kept_issues) @@ -592,19 +1188,25 @@ def deduplicate_cross_batch_issues(issues: List[CodeReviewIssue]) -> List[CodeRe deduped = [] for issue in issues: - if any( - issues_are_conservative_duplicates(issue, existing) - for existing in deduped - ): - issue_data = _issue_payload(issue) - logger.info( - "Cross-batch dedup: suppressed anchored duplicate at %s:%s", - issue_data.get("file", ""), - issue_data.get("line", ""), - ) + duplicate_index = next(( + index + for index, existing in enumerate(deduped) + if issues_are_conservative_duplicates(issue, existing) + ), None) + if duplicate_index is None: + deduped.append(issue) continue - deduped.append(issue) - + deduped[duplicate_index] = _merge_duplicate_issues( + deduped[duplicate_index], + issue, + ) + issue_data = _issue_payload(issue) + logger.info( + "Cross-batch dedup: merged exact root identity at %s:%s", + issue_data.get("file", ""), + issue_data.get("line", ""), + ) + return deduped async def reconcile_previous_issues( diff --git a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py index 446bda7f5..cdb5576b9 100644 --- a/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py +++ b/python-ecosystem/inference-orchestrator/src/service/review/orchestrator/stage_3_aggregation.py @@ -3,6 +3,7 @@ """ import json import logging +import re from typing import Any, Dict, List, Optional from model.dtos import ReviewRequestDto @@ -31,6 +32,7 @@ async def execute_stage_3_aggregation( fallback_llm=None, ) -> Dict[str, Any]: stage_1_json = _summarize_issues_for_stage_3(stage_1_issues) + verification_issues = _stage_3_verification_issue_map(stage_1_issues) stage_2_json = stage_2_results.model_dump_json(indent=2) plan_summary = _summarize_plan_for_stage_3(plan) @@ -49,7 +51,7 @@ async def execute_stage_3_aggregation( additions = processed_diff.total_additions if processed_diff else 0 deletions = processed_diff.total_deletions if processed_diff else 0 - target_branch = request.targetBranchName or "" + review_revision = _review_revision(request) prompt = PromptBuilder.build_stage_3_aggregation_prompt( repo_slug=request.projectVcsRepoSlug, @@ -69,22 +71,38 @@ async def execute_stage_3_aggregation( or "No task context available." ), use_mcp_tools=use_mcp_tools, - target_branch=target_branch, + review_revision=review_revision, ) - if use_mcp_tools and mcp_client and target_branch: + if use_mcp_tools and mcp_client and review_revision: return await _stage_3_with_mcp( llm, request, prompt, mcp_client, - target_branch, + review_revision, + verification_issues, fallback_llm=fallback_llm, ) + if use_mcp_tools and mcp_client and not review_revision: + logger.warning( + "[Stage 3] MCP verification skipped: no immutable reviewed commit " + "hash was supplied" + ) + return await _invoke_stage_3_report(llm, prompt, fallback_llm=fallback_llm) +def _review_revision(request: ReviewRequestDto) -> str: + """Return an immutable review revision; never substitute a moving branch.""" + for field in ("currentCommitHash", "commitHash"): + value = getattr(request, field, None) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + async def _invoke_stage_3_report(llm, prompt: str, fallback_llm=None) -> Dict[str, Any]: response = await llm.ainvoke(prompt) if _response_finished_by_length(response) and fallback_llm is not None and fallback_llm is not llm: @@ -133,26 +151,104 @@ def _summarize_issues_for_stage_3(issues: List[CodeReviewIssue]) -> str: "By category: " + ", ".join(f"{k}: {v}" for k, v in sorted(category_counts.items())), ] - priority_order = {'CRITICAL': 0, 'HIGH': 1, 'MEDIUM': 2, 'LOW': 3, 'INFO': 4} - ranked = sorted(issues, key=lambda i: priority_order.get(i.severity.upper(), 5)) - top_n = ranked[:10] - if top_n: - lines.append("\nTop findings (issue IDs are for internal reference):") - for i, issue in enumerate(top_n, 1): - issue_id = getattr(issue, 'id', '') or '' - title = getattr(issue, 'title', '') or '' - title_part = f" {title} —" if title else "" - lines.append(f" {i}. [id={issue_id}] [{issue.severity}] {issue.file}:{title_part} {issue.reason[:120]}") - - if issues: - all_ids = [getattr(i, 'id', '') or '' for i in issues] - all_ids = [i for i in all_ids if i] - if all_ids: - lines.append(f"\nAll issue IDs: {', '.join(all_ids)}") + # Stage 3 can verify any finding, including fresh ones without database IDs. + # Emit a semantically compact record for every active issue instead of + # clipping details to a top-ten list. + records = [ + _stage_3_verification_record(verification_id, issue) + for verification_id, issue in _stage_3_verification_issue_map(issues).items() + ] + lines.append("\nComplete verification records (JSON):") + lines.append(json.dumps(records, ensure_ascii=False, separators=(",", ":"))) return "\n".join(lines) +def _safe_issue_field(issue: CodeReviewIssue, name: str) -> Any: + value = getattr(issue, name, "") + if value is None: + return "" + if value.__class__.__module__.startswith("unittest.mock"): + return "" + return value + + +def _stage_3_verification_issue_map( + issues: List[CodeReviewIssue], +) -> Dict[str, CodeReviewIssue]: + active = [ + issue + for issue in issues + if getattr(issue, "isResolved", False) is not True + ] + return { + f"issue_{index}": issue + for index, issue in enumerate(active) + } + + +_RELATED_LOCATIONS_RE = re.compile( + r"(?im)^\s*(?:[*_]{1,2})?also affects\s*:(?:[*_]{1,2})?\s*(.+)$" +) + + +def _stage_3_reason_brief(issue: CodeReviewIssue) -> str: + """Remove exact repetition while preserving every substantive paragraph.""" + reason = str(_safe_issue_field(issue, "reason") or "").strip() + if not reason: + return "" + paragraphs = [ + paragraph.strip() + for paragraph in re.split(r"\n\s*\n", reason) + if paragraph.strip() + ] + title = str(_safe_issue_field(issue, "title") or "").strip().casefold() + selected: List[str] = [] + seen: set[str] = set() + for paragraph in paragraphs: + normalized = " ".join(paragraph.strip("*_# ").casefold().split()) + if normalized == title: + continue + if normalized in seen: + continue + selected.append(paragraph) + seen.add(normalized) + return "\n\n".join(selected) if selected else reason + + +def _normalized_related_locations(issue: CodeReviewIssue) -> List[str]: + values = _safe_issue_field(issue, "relatedLocations") or [] + locations = list(values) if isinstance(values, (list, tuple, set)) else [] + reason = str(_safe_issue_field(issue, "reason") or "") + for match in _RELATED_LOCATIONS_RE.finditer(reason): + locations.extend(match.group(1).split(",")) + return sorted({ + str(value).strip() + for value in locations + if str(value).strip() + }) + + +def _stage_3_verification_record( + verification_id: str, + issue: CodeReviewIssue, +) -> Dict[str, Any]: + return { + "verification_id": verification_id, + "original_id": str(_safe_issue_field(issue, "id") or ""), + "severity": str(_safe_issue_field(issue, "severity") or ""), + "category": str(_safe_issue_field(issue, "category") or ""), + "file": str(_safe_issue_field(issue, "file") or ""), + "line": _safe_issue_field(issue, "line") or 0, + "title": str(_safe_issue_field(issue, "title") or ""), + "reason": _stage_3_reason_brief(issue), + "exact_source_anchor": str( + _safe_issue_field(issue, "codeSnippet") or "" + ), + "related_locations": _normalized_related_locations(issue), + } + + def _summarize_plan_for_stage_3(plan: ReviewPlan) -> str: lines = [] total_files = sum(len(g.files) for g in plan.file_groups) @@ -194,26 +290,151 @@ def _extract_dismissed_issues(content: str) -> tuple: try: dismissed = json.loads(match.group(1)) if not isinstance(dismissed, list): - logger.warning(f"[Stage 3] DISMISSED_ISSUES was not a list: {match.group(1)}") + logger.warning( + "[Stage 3] DISMISSED_ISSUES was not a list: %s", + match.group(1), + ) return content, [] dismissed = [str(d) for d in dismissed if d] - logger.info(f"[Stage 3] MCP verification dismissed {len(dismissed)} issues: {dismissed}") + logger.info( + "[Stage 3] MCP verification requested dismissal of %d issues: %s", + len(dismissed), + dismissed, + ) clean_report = content[:match.start()].rstrip() + content[match.end():] return clean_report.strip(), dismissed - except (json.JSONDecodeError, TypeError) as e: - logger.warning(f"[Stage 3] Failed to parse DISMISSED_ISSUES: {e}") + except (json.JSONDecodeError, TypeError) as exc: + logger.warning("[Stage 3] Failed to parse DISMISSED_ISSUES: %s", exc) return content, [] +def _location_file_path(location: str) -> str: + normalized = str(location or "").strip().replace("\\", "/").lstrip("/") + if not normalized: + return "" + path, separator, possible_line = normalized.rpartition(":") + if separator and possible_line.isdigit(): + return path + return normalized + + +def _location_line(location: str) -> int: + normalized = str(location or "").strip().replace("\\", "/") + _, separator, possible_line = normalized.rpartition(":") + if separator and possible_line.isdigit(): + return int(possible_line) + return 0 + + +def _required_verification_locations( + issue: CodeReviewIssue, +) -> set[tuple[str, int]]: + primary_path = _location_file_path( + str(_safe_issue_field(issue, "file") or "") + ) + try: + primary_line = int(_safe_issue_field(issue, "line") or 0) + except (TypeError, ValueError): + primary_line = 0 + locations = {(primary_path, max(0, primary_line))} + locations.update( + (_location_file_path(location), _location_line(location)) + for location in _normalized_related_locations(issue) + ) + return {(path, line) for path, line in locations if path} + + +def _mcp_read_covers_location( + entry: Dict[str, Any], + verification_id: str, + file_path: str, + line: int, + review_revision: str, +) -> bool: + args = entry.get("args", {}) + if not ( + entry.get("tool") == "getBranchFileContent" + and entry.get("success") is True + and entry.get("evidence_valid") is True + and str(args.get("verificationId") or "") == verification_id + and _location_file_path(str(args.get("filePath") or "")) == file_path + and str(args.get("branch") or "") == review_revision + ): + return False + if entry.get("evidence_complete_file") is True: + return True + if line <= 0: + return False + try: + start_line = int(entry.get("evidence_start_line") or 0) + end_line = int(entry.get("evidence_end_line") or 0) + except (TypeError, ValueError): + return False + return start_line > 0 and start_line <= line <= end_line + + +def _validated_mcp_dismissals( + requested_ids: List[str], + issue_by_verification_id: Dict[str, CodeReviewIssue], + executor: McpToolExecutor, + review_revision: str, +) -> List[str]: + """Accept dismissals only when every affected anchor has bound evidence.""" + validated: List[str] = [] + for verification_id in requested_ids: + issue = issue_by_verification_id.get(verification_id) + if issue is None: + logger.warning( + "[Stage 3] Ignoring dismissal for unknown verification ID %s", + verification_id, + ) + continue + required_locations = _required_verification_locations(issue) + missing_locations = { + location + for location in required_locations + if not any( + _mcp_read_covers_location( + entry, + verification_id, + location[0], + location[1], + review_revision, + ) + for entry in executor.call_log + ) + } + if not required_locations or missing_locations: + logger.warning( + "[Stage 3] Keeping %s: dismissal lacks successful reviewed-revision " + "evidence for %s", + verification_id, + sorted( + f"{path}:{line}" if line > 0 else path + for path, line in missing_locations + ), + ) + continue + validated.append(verification_id) + return validated + + async def _stage_3_with_mcp( llm, request: ReviewRequestDto, prompt: str, mcp_client, - target_branch: str, + review_revision: str, + issue_by_verification_id: Dict[str, CodeReviewIssue], fallback_llm=None, ) -> Dict[str, Any]: - executor = McpToolExecutor(mcp_client, request, stage="stage_3") + executor = McpToolExecutor( + mcp_client, + request, + stage="stage_3", + review_revision=review_revision, + verification_issues=issue_by_verification_id, + ) tool_defs = executor.get_tool_definitions() max_iterations = 15 @@ -234,7 +455,8 @@ async def _stage_3_with_mcp( request, prompt, mcp_client, - target_branch, + review_revision, + issue_by_verification_id, ) content = extract_llm_response_text(response) logger.info( @@ -242,7 +464,24 @@ async def _stage_3_with_mcp( f"{executor.call_count} verification calls" ) report, dismissed = _extract_dismissed_issues(content) - return {"report": report, "dismissed_issue_ids": dismissed} + validated = _validated_mcp_dismissals( + dismissed, + issue_by_verification_id, + executor, + review_revision, + ) + return { + "report": report, + "dismissed_issue_ids": [ + str(_safe_issue_field(issue_by_verification_id[key], "id") or "") + for key in validated + if str(_safe_issue_field(issue_by_verification_id[key], "id") or "") + ], + "dismissed_issue_keys": validated, + "dismissed_issue_object_ids": [ + id(issue_by_verification_id[key]) for key in validated + ], + } for tc in tool_calls: tool_result = await executor.execute_tool(tc["name"], tc["args"]) diff --git a/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_mcp.py b/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_mcp.py index ccf63f004..51080a808 100644 --- a/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_mcp.py +++ b/python-ecosystem/inference-orchestrator/src/utils/prompts/constants_mcp.py @@ -21,20 +21,29 @@ STAGE_3_MCP_VERIFICATION_SECTION = """ ## Issue Re-verification (Optional) Before producing the final report, you may verify HIGH/CRITICAL issues that seem uncertain -by reading actual file content from the repository. +by reading actual file content from the exact reviewed PR revision. Available tools: -- **getBranchFileContent(branch, filePath)** — Read a file to verify an issue's existence +- **getBranchFileContent(filePath, verificationId)** — Read an anchor-centred + source window; the host supplies the exact reviewed commit - **getPullRequestComments(pullRequestId)** — Read PR comments for additional context RULES: 1. You have a MAXIMUM of {max_calls} verification calls total. 2. Only verify issues you are UNCERTAIN about — do not verify every issue. -3. Focus on HIGH and CRITICAL severity issues. -4. If verification reveals a false positive, note its ID for dismissal. -5. After verification, produce the final executive summary. - -TARGET BRANCH: {target_branch} +3. Prioritize HIGH and CRITICAL severity, but you may verify a lower-severity + finding when its correctness materially affects the final report. +4. Use the Verification ID from the complete verification-record list, including + records whose persisted Original ID is empty. +5. Pass that same Verification ID in every file-content call. The host binds the + returned source window to the finding and verifies that it covers its line. +6. If a finding has related_locations, every affected location must be read with + the same Verification ID before dismissing the consolidated root finding. +7. If verification reveals a false positive, note its Verification ID for dismissal. +8. Missing, failed, partial, or ambiguous evidence means KEEP the finding. +9. After verification, produce the final executive summary. + +REVIEWED REVISION: {review_revision} PR ID: {pr_id} ## False Positive Dismissal @@ -42,11 +51,13 @@ positives, append an HTML comment at the very end of your response with the IDs of issues that should be removed from the issue list: - + RULES for dismissal: -- Only dismiss issues you VERIFIED as false positives via tool calls (read the actual code). +- Only dismiss issues you VERIFIED as false positives via successful file-content + tool calls against the reviewed revision. - Do NOT dismiss issues based on guessing — you must have read the relevant file. -- Architecture observations reported as HIGH severity bugs can be dismissed if they have no runtime impact. +- Do not dismiss a concrete architecture/maintainability defect merely because it + has no immediate runtime crash; verify the claim as written. - If no issues should be dismissed, omit the DISMISSED_ISSUES comment entirely. """ diff --git a/python-ecosystem/inference-orchestrator/src/utils/prompts/prompt_builder.py b/python-ecosystem/inference-orchestrator/src/utils/prompts/prompt_builder.py index bfb84e6b1..18f0ca582 100644 --- a/python-ecosystem/inference-orchestrator/src/utils/prompts/prompt_builder.py +++ b/python-ecosystem/inference-orchestrator/src/utils/prompts/prompt_builder.py @@ -333,7 +333,7 @@ def build_stage_3_aggregation_prompt( incremental_context: str = "", task_context: str = "No task context available.", use_mcp_tools: bool = False, - target_branch: str = "", + review_revision: str = "", ) -> str: """ Build prompt for Stage 3: Aggregation & Final Report. @@ -356,12 +356,12 @@ def build_stage_3_aggregation_prompt( ) # Conditionally append MCP verification instructions - if use_mcp_tools and target_branch: + if use_mcp_tools and review_revision: from service.review.orchestrator.mcp_tool_executor import McpToolExecutor max_calls = McpToolExecutor.STAGE_CONFIG["stage_3"]["max_calls"] prompt += STAGE_3_MCP_VERIFICATION_SECTION.format( max_calls=max_calls, - target_branch=target_branch, + review_revision=review_revision, pr_id=pr_id ) diff --git a/python-ecosystem/inference-orchestrator/tests/test_inference_policy.py b/python-ecosystem/inference-orchestrator/tests/test_inference_policy.py index 806858a79..9dde882fd 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_inference_policy.py +++ b/python-ecosystem/inference-orchestrator/tests/test_inference_policy.py @@ -53,21 +53,26 @@ def _full_profile(): ) -def test_llm_dedup_is_disabled_by_default(): - assert should_use_llm_dedup(_full_profile(), 20) is False +def test_grouped_llm_dedup_is_enabled_by_default(): + assert should_use_llm_dedup(_full_profile(), 20) is True + assert should_use_llm_dedup(_fast_profile(), 2) is True -def test_llm_dedup_requires_explicit_opt_in(monkeypatch): +def test_llm_dedup_can_be_disabled_explicitly(monkeypatch): monkeypatch.setattr( "service.review.orchestrator.inference_policy.LLM_DEDUP_ENABLED", - True, + False, ) - assert should_use_llm_dedup(_full_profile(), 20) is True + assert should_use_llm_dedup(_full_profile(), 20) is False assert should_use_llm_dedup(_full_profile(), 1) is False assert should_use_llm_dedup(_fast_profile(), 2) is False +def test_llm_dedup_skips_single_finding(): + assert should_use_llm_dedup(_full_profile(), 1) is False + + def test_task_context_forces_stage_2_in_fast_check(): request = _request(taskContext={"task_key": "PROJ-123"}) plan = ReviewPlan(analysis_summary="plan", file_groups=[], cross_file_concerns=[]) diff --git a/python-ecosystem/inference-orchestrator/tests/test_mcp_tool_executor.py b/python-ecosystem/inference-orchestrator/tests/test_mcp_tool_executor.py index 7b27c8d38..5810b538e 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_mcp_tool_executor.py +++ b/python-ecosystem/inference-orchestrator/tests/test_mcp_tool_executor.py @@ -5,6 +5,7 @@ import pytest from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock +from model.output_schemas import CodeReviewIssue from service.review.orchestrator.mcp_tool_executor import McpToolExecutor @@ -60,6 +61,9 @@ async def test_successful_call(self): result = await e.execute_tool("getBranchFileContent", {"filePath": "a.py", "branch": "main"}) assert result == "file content here" assert e.call_count == 1 + assert e.call_log[0]["result_chars"] == len("file content here") + assert e.call_log[0]["evidence_valid"] is True + assert e.call_log[0]["evidence_complete_file"] is True @pytest.mark.asyncio(loop_scope="function") async def test_call_failure(self): @@ -71,6 +75,23 @@ async def test_call_failure(self): assert len(e.call_log) == 1 assert e.call_log[0]["success"] is False + @pytest.mark.asyncio(loop_scope="function") + async def test_mcp_error_result_is_not_valid_file_evidence(self): + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock(return_value=SimpleNamespace( + content=[SimpleNamespace(text="Error executing tool: unavailable")], + isError=True, + )) + e = McpToolExecutor(mock_client, _make_request(), "stage_1") + + await e.execute_tool( + "getBranchFileContent", + {"filePath": "a.py", "branch": "main"}, + ) + + assert e.call_log[0]["success"] is False + assert e.call_log[0]["evidence_valid"] is False + @pytest.mark.asyncio(loop_scope="function") async def test_prefills_workspace(self): mock_client = MagicMock() @@ -83,6 +104,133 @@ async def test_prefills_workspace(self): assert call_args[1]["workspace"] == "ws" assert call_args[1]["repoSlug"] == "repo" + @pytest.mark.asyncio(loop_scope="function") + async def test_stage_3_pins_file_reads_to_reviewed_revision(self): + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock( + return_value=SimpleNamespace(content=[SimpleNamespace(text="source")]) + ) + e = McpToolExecutor( + mock_client, + _make_request(), + "stage_3", + review_revision="commit-abc", + ) + + await e.execute_tool( + "getBranchFileContent", + {"filePath": "a.py", "branch": "main"}, + ) + + call_args = mock_client.session.call_tool.call_args[0][1] + assert call_args["branch"] == "commit-abc" + assert e.call_log[0]["args"]["branch"] == "commit-abc" + + @pytest.mark.asyncio(loop_scope="function") + async def test_stage_3_requests_window_around_bound_finding_anchor(self): + issue = CodeReviewIssue( + file="src/a.py", line=500, severity="HIGH", category="BUG_RISK", + reason="Concrete defect.", suggestedFixDescription="Fix it.", + ) + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock( + return_value=SimpleNamespace(content=[SimpleNamespace( + text=( + '{"fileContent":"source","startLine":420,' + '"endLine":580,"totalLines":1000,"completeFile":false}' + ) + )]) + ) + e = McpToolExecutor( + mock_client, + _make_request(), + "stage_3", + review_revision="commit-abc", + verification_issues={"issue_0": issue}, + ) + + await e.execute_tool("getBranchFileContent", { + "filePath": "src/a.py", + "branch": "main", + "verificationId": "issue_0", + }) + + call_args = mock_client.session.call_tool.await_args.args[1] + assert call_args["startLine"] == 420 + assert call_args["endLine"] == 580 + assert e.call_log[0]["evidence_valid"] is True + assert e.call_log[0]["evidence_structured"] is True + assert e.call_log[0]["evidence_start_line"] == 420 + assert e.call_log[0]["evidence_end_line"] == 580 + + @pytest.mark.asyncio(loop_scope="function") + async def test_stage_3_raw_adapter_response_is_bound_to_requested_window(self): + issue = CodeReviewIssue( + file="src/a.py", line=500, severity="HIGH", category="BUG_RISK", + reason="Concrete defect.", suggestedFixDescription="Fix it.", + ) + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock( + return_value=SimpleNamespace( + content=[SimpleNamespace(text="raw source window")] + ) + ) + e = McpToolExecutor( + mock_client, + _make_request(), + "stage_3", + review_revision="commit-abc", + verification_issues={"issue_0": issue}, + ) + + await e.execute_tool("getBranchFileContent", { + "filePath": "src/a.py", + "branch": "main", + "verificationId": "issue_0", + }) + + assert e.call_log[0]["evidence_valid"] is True + assert e.call_log[0]["evidence_structured"] is False + assert e.call_log[0]["evidence_complete_file"] is False + assert e.call_log[0]["evidence_start_line"] == 420 + assert e.call_log[0]["evidence_end_line"] == 580 + + @pytest.mark.asyncio(loop_scope="function") + @pytest.mark.parametrize("tool_text", [ + '{"error":"file not found"}', + 'Error executing tool: permission denied', + ( + '{"fileContent":"[CodeCrow Filter: file too large, omitted]",' + '"completeFile":false}' + ), + '{"fileContent":"","completeFile":true}', + ]) + async def test_error_empty_and_filtered_results_are_not_source_evidence( + self, + tool_text, + ): + mock_client = MagicMock() + mock_client.session.call_tool = AsyncMock( + return_value=SimpleNamespace( + content=[SimpleNamespace(text=tool_text)] + ) + ) + e = McpToolExecutor( + mock_client, + _make_request(), + "stage_3", + review_revision="commit-abc", + ) + + await e.execute_tool("getBranchFileContent", { + "filePath": "src/a.py", + "branch": "commit-abc", + "verificationId": "issue_0", + }) + + assert e.call_log[0]["success"] is True + assert e.call_log[0]["evidence_valid"] is False + # ── get_tool_definitions ───────────────────────────────────── @@ -99,6 +247,31 @@ def test_stage_3_definitions(self): names = {d["function"]["name"] for d in defs} assert "getBranchFileContent" in names assert "getPullRequestComments" in names + file_tool = next( + definition for definition in defs + if definition["function"]["name"] == "getBranchFileContent" + ) + assert "verificationId" in file_tool["function"]["parameters"]["required"] + assert "branch" not in file_tool["function"]["parameters"]["required"] + + def test_related_location_in_reason_drives_a_bound_source_window(self): + issue = CodeReviewIssue( + file="src/a.py", line=10, severity="HIGH", category="BUG_RISK", + reason=( + "One root cause.\n\n" + "Also affects: src/b.py:700, src/c.py:900" + ), + suggestedFixDescription="Fix it.", + ) + executor = McpToolExecutor( + MagicMock(), _make_request(), "stage_3", + review_revision="commit-abc", + verification_issues={"issue_0": issue}, + ) + + assert executor._verification_line_for_path( + "issue_0", "src/b.py" + ) == 700 # ── Properties ─────────────────────────────────────────────── diff --git a/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py index 496e87c38..5e0bf4e7f 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_orchestrator_helpers.py @@ -34,6 +34,7 @@ ) from model.output_schemas import CodeReviewIssue from model.dtos import ReviewRequestDto +from service.review.candidate_ledger import CandidateEvidenceLedger @pytest.fixture @@ -703,6 +704,52 @@ def test_protected_history_suppresses_exact_anchored_duplicate(self): assert retained_fresh == [] + def test_superseded_history_close_copy_does_not_republish_rejected_candidate(self): + retained_history = CodeReviewIssue( + id="100", file="a.py", line=10, severity="MEDIUM", + category="BUG_RISK", title="Missing workspace guard", + reason="The update path does not validate workspace ownership.", + suggestedFixDescription="Add the ownership guard.", + ) + superseded_history = CodeReviewIssue( + id="200", file="a.py", line=11, severity="HIGH", + category="SECURITY", title="Missing workspace guard", + reason="The update path does not validate workspace ownership.", + suggestedFixDescription="Add the ownership guard.", + ) + ledger = CandidateEvidenceLedger() + ledger.register( + superseded_history, + stage="stage_1", + source_key="batch-1", + review_unit_ids=["unit-1"], + prompt_hunk_ids=["hunk-1"], + prompt_digest="sha256:" + ("0" * 64), + ) + before = [retained_history, superseded_history] + + result = _deduplicate_cross_batch_issues_preserving_lifecycle( + before, + {"100", "200"}, + ) + + active, resolved = _partition_issue_lifecycle(result) + assert len(active) == 1 + assert active[0].id == "100" + assert len(resolved) == 1 + assert resolved[0].id == "200" + assert resolved[0] is not superseded_history + assert superseded_history.isResolved is False + + ledger.reject_removed( + before, + result, + gate="deduplication", + code="cross_batch_duplicate", + ) + ledger.publish(result) + ledger.assert_terminal() + class TestSerializeIssueForClient: def test_active_issue_does_not_serialize_resolution_metadata(self): @@ -795,3 +842,50 @@ def test_resolves_historical_open_issue_and_drops_fresh_candidate(self): assert historical.resolutionExplanation == historical.resolutionReason assert resolved_count == 1 assert dropped_count == 1 + + def test_object_identity_drops_verified_fresh_issue_without_database_id(self): + fresh = CodeReviewIssue( + file="a.py", line=10, severity="MEDIUM", category="BUG_RISK", + reason="Fresh false positive.", suggestedFixDescription="No fix.", + ) + unaffected = CodeReviewIssue( + file="a.py", line=20, severity="MEDIUM", category="BUG_RISK", + reason="Independent real issue.", suggestedFixDescription="Fix it.", + ) + + retained, resolved_count, dropped_count = _apply_stage_3_dismissals( + [fresh, unaffected], + set(), + set(), + dismissed_object_ids={id(fresh)}, + ) + + assert retained == [unaffected] + assert resolved_count == 0 + assert dropped_count == 1 + + def test_object_identity_does_not_touch_resolved_record_with_same_id(self): + active = CodeReviewIssue( + id="12524", file="a.py", line=10, severity="MEDIUM", + category="BUG_RISK", reason="Unsupported active finding.", + suggestedFixDescription="No fix.", + ) + prior_resolution = CodeReviewIssue( + id="12524", file="a.py", line=10, severity="INFO", + category="BUG_RISK", reason="Historical lifecycle update.", + suggestedFixDescription="Already fixed.", isResolved=True, + resolutionReason="Previously closed.", + ) + + retained, resolved_count, dropped_count = _apply_stage_3_dismissals( + [active, prior_resolution], + {"12524"}, + {"12524"}, + dismissed_object_ids={id(active)}, + ) + + assert retained == [active, prior_resolution] + assert active.isResolved is True + assert prior_resolution.resolutionReason == "Previously closed." + assert resolved_count == 1 + assert dropped_count == 0 diff --git a/python-ecosystem/inference-orchestrator/tests/test_prompt_builder.py b/python-ecosystem/inference-orchestrator/tests/test_prompt_builder.py index e6736ff9d..310ae2131 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_prompt_builder.py +++ b/python-ecosystem/inference-orchestrator/tests/test_prompt_builder.py @@ -320,3 +320,16 @@ def test_with_task_context(self): ) assert "PROJ-3" in result assert "task-coverage" in result + + def test_mcp_verification_uses_reviewed_revision_and_verification_ids(self): + result = PromptBuilder.build_stage_3_aggregation_prompt( + repo_slug="r", pr_id="7", author="d", pr_title="T", + total_files=1, additions=1, deletions=0, + stage_0_plan="p", stage_1_issues_json="[]", + stage_2_findings_json="[]", recommendation="APPROVE", + use_mcp_tools=True, review_revision="commit-abc", + ) + + assert "REVIEWED REVISION: commit-abc" in result + assert "Verification ID" in result + assert 'DISMISSED_ISSUES: ["issue_0", "issue_3"]' in result diff --git a/python-ecosystem/inference-orchestrator/tests/test_reconciliation.py b/python-ecosystem/inference-orchestrator/tests/test_reconciliation.py index 149a8d2aa..f756a44f0 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_reconciliation.py +++ b/python-ecosystem/inference-orchestrator/tests/test_reconciliation.py @@ -15,6 +15,7 @@ format_previous_issues_for_batch, deduplicate_final_issues, deduplicate_cross_batch_issues, + issues_are_semantic_dedup_candidates, _build_batches, reconcile_previous_issues, ) @@ -302,7 +303,54 @@ def test_same_source_snippet_survives_line_drift_and_deduplicates(self): assert len(deduplicate_final_issues(issues)) == 1 - def test_exact_plugin_proof_deduplicates_across_prose_and_anchors(self): + def test_exact_history_recreation_keeps_identity_and_refreshes_anchor(self): + historical = _make_issue( + id="3989", + file="src/service.py", + line=1, + title="Unbounded retry loop can exhaust workers", + category="CODE_QUALITY", + severity="MEDIUM", + reason="The retry loop has no terminal attempt limit.", + codeSnippet="class RetryService:", + ) + recreated = _make_issue( + file="src/service.py", + line=1233, + title="Unbounded retry loop can exhaust workers", + category="BUG_RISK", + severity="HIGH", + reason="The retry loop has no terminal attempt limit.", + codeSnippet="while should_retry(response):", + ) + + result = deduplicate_final_issues([historical, recreated]) + + assert len(result) == 1 + assert result[0].id == "3989" + assert result[0].line == 1233 + assert result[0].codeSnippet == "while should_retry(response):" + assert result[0].severity == "HIGH" + + def test_distinct_historical_duplicates_keep_both_concrete_locations(self): + first = _make_issue( + id="3989", file="src/service.py", line=10, + title="Unbounded retry loop can exhaust workers", + reason="The retry loop has no terminal attempt limit.", + ) + second = _make_issue( + id="3990", file="src/service.py", line=20, + title="Unbounded retry loop can exhaust workers", + reason="The retry loop has no terminal attempt limit.", + ) + + result = deduplicate_final_issues([second, first]) + + assert len(result) == 1 + assert result[0].id == "3989" + assert result[0].relatedLocations == ["src/service.py:20"] + + def test_exact_plugin_proof_is_a_semantic_candidate_not_a_delete_key(self): issues = [ _make_issue( file="app/code/Vendor/Module/etc/di.xml", @@ -324,7 +372,8 @@ def test_exact_plugin_proof_deduplicates_across_prose_and_anchors(self): ), ] - assert deduplicate_final_issues(issues) == [issues[0]] + assert deduplicate_final_issues(issues) == issues + assert issues_are_semantic_dedup_candidates(issues[0], issues[1]) def test_distinct_plugin_proofs_remain_distinct(self): issues = [ diff --git a/python-ecosystem/inference-orchestrator/tests/test_reconciliation_full.py b/python-ecosystem/inference-orchestrator/tests/test_reconciliation_full.py index e69c352db..991a1763d 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_reconciliation_full.py +++ b/python-ecosystem/inference-orchestrator/tests/test_reconciliation_full.py @@ -1,12 +1,20 @@ """Extended tests for reconciliation: _format_issues_for_prompt, _build_batches, _dedup_batch_with_llm.""" import pytest from unittest.mock import MagicMock, AsyncMock, patch +from model.output_schemas import ( + CodeReviewIssue, + SemanticDeduplicationDecision, + SemanticDuplicateGroup, +) from service.review.orchestrator.reconciliation import ( _format_issues_for_prompt, _build_batches, + _build_semantic_dedup_batches, _dedup_batch_with_llm, + _semantic_candidate_groups, deduplicate_final_issues_llm, deduplicate_final_issues, + issues_are_semantic_dedup_candidates, ) @@ -30,6 +38,22 @@ def _make_issue(file="a.py", line=10, severity="HIGH", category="BUG_RISK", return issue +def _real_issue(file="a.py", line=10, severity="HIGH", category="BUG_RISK", + title="Issue", reason="Something wrong", issue_id=None, + code_snippet=""): + return CodeReviewIssue( + id=issue_id, + file=file, + line=line, + severity=severity, + category=category, + title=title, + reason=reason, + suggestedFixDescription="Fix the root cause.", + codeSnippet=code_snippet, + ) + + # ── _format_issues_for_prompt ───────────────────────────────── @@ -86,32 +110,91 @@ def test_empty(self): class TestDedupBatchWithLlm: @pytest.mark.asyncio(loop_scope="function") - async def test_keeps_selected_indices(self): - from model.output_schemas import DeduplicatedIssueList + async def test_merges_only_explicit_high_confidence_group(self): llm = MagicMock() structured = MagicMock() structured.ainvoke = AsyncMock( - return_value=DeduplicatedIssueList(kept_indices=[0, 2]) + return_value=SemanticDeduplicationDecision(duplicate_groups=[ + SemanticDuplicateGroup( + keeper_index=0, + duplicate_indices=[1], + confidence="HIGH", + rationale="One missing authorization guard.", + ) + ]) ) llm.with_structured_output.return_value = structured - issues = [_make_issue(file=f"f{i}.py") for i in range(3)] - result = await _dedup_batch_with_llm(llm, issues) - assert len(result) == 2 + issues = [ + _real_issue( + line=10, + title="Authorization guard missing from update path", + reason="The update path writes account data without checking the workspace role.", + ), + _real_issue( + line=40, + title="Authorization guard missing from update path", + reason="Account data is written by this update path before the workspace role is checked.", + ), + ] + result = await _dedup_batch_with_llm( + llm, + issues, + {0: "candidate_0", 1: "candidate_0"}, + ) + assert len(result) == 1 + assert result[0].relatedLocations == ["a.py:40"] @pytest.mark.asyncio(loop_scope="function") async def test_invalid_indices_keeps_all(self): - from model.output_schemas import DeduplicatedIssueList llm = MagicMock() structured = MagicMock() structured.ainvoke = AsyncMock( - return_value=DeduplicatedIssueList(kept_indices=[99]) + return_value=SemanticDeduplicationDecision(duplicate_groups=[ + SemanticDuplicateGroup( + keeper_index=0, + duplicate_indices=[99], + confidence="HIGH", + rationale="Malformed index.", + ) + ]) ) llm.with_structured_output.return_value = structured - issues = [_make_issue() for _ in range(2)] + issues = [ + _real_issue(line=10, reason="First independent problem."), + _real_issue(line=20, reason="Second independent problem."), + ] + result = await _dedup_batch_with_llm( + llm, + issues, + {0: "candidate_0", 1: "candidate_0"}, + ) + assert result == issues + + @pytest.mark.asyncio(loop_scope="function") + async def test_uncertain_decision_keeps_all(self): + llm = MagicMock() + structured = MagicMock() + structured.ainvoke = AsyncMock( + return_value=SemanticDeduplicationDecision(duplicate_groups=[ + SemanticDuplicateGroup( + keeper_index=0, + duplicate_indices=[1], + confidence="MEDIUM", + rationale="Possibly related.", + ) + ]) + ) + llm.with_structured_output.return_value = structured + issues = [ + _real_issue(line=10, reason="First problem."), + _real_issue(line=20, reason="Second problem."), + ] + result = await _dedup_batch_with_llm(llm, issues) - assert len(result) == 2 # All kept as fallback + + assert result == issues @pytest.mark.asyncio(loop_scope="function") async def test_exception_falls_back(self): @@ -120,10 +203,12 @@ async def test_exception_falls_back(self): structured.ainvoke = AsyncMock(side_effect=Exception("fail")) llm.with_structured_output.return_value = structured - issues = [_make_issue() for _ in range(2)] + issues = [ + _real_issue(line=10, reason="First independent problem."), + _real_issue(line=20, reason="Second independent problem."), + ] result = await _dedup_batch_with_llm(llm, issues) - # Falls back to algorithmic dedup - assert len(result) >= 1 + assert result == issues # ── deduplicate_final_issues ────────────────────────────────── @@ -152,12 +237,112 @@ def test_exact_duplicates(self): class TestDeduplicateFinalIssuesLlm: @pytest.mark.asyncio(loop_scope="function") - async def test_small_set_uses_algorithmic(self): - # Less than the batch threshold → should just use algorithmic - issues = [_make_issue(file=f"f{i}.py") for i in range(2)] + async def test_non_candidates_do_not_use_model_tokens(self): + issues = [ + _real_issue(file="a.py", title="SQL injection", reason="Raw SQL uses user input."), + _real_issue(file="b.py", title="Cache stampede", reason="Cache misses fan out."), + ] llm = MagicMock() result = await deduplicate_final_issues_llm(llm, issues) - assert len(result) >= 1 + assert result == issues + llm.with_structured_output.assert_not_called() + + @pytest.mark.asyncio(loop_scope="function") + async def test_sends_complete_candidate_evidence_but_not_singletons(self): + long_evidence = "Evidence marker " + ("complete-context " * 900) + candidate_a = _real_issue( + line=10, + title="Workspace authorization is missing from account update", + reason=long_evidence + "role check is absent", + ) + candidate_b = _real_issue( + line=50, + title="Workspace authorization is missing from account update", + reason=long_evidence + "role validation is absent", + ) + singleton = _real_issue( + file="different.py", + title="Independent resource leak", + reason="SINGLETON-MUST-NOT-BE-SENT", + ) + llm = MagicMock() + structured = MagicMock() + structured.ainvoke = AsyncMock( + return_value=SemanticDeduplicationDecision(duplicate_groups=[]) + ) + llm.with_structured_output.return_value = structured + + result = await deduplicate_final_issues_llm( + llm, + [candidate_a, singleton, candidate_b], + ) + + assert result == [candidate_a, singleton, candidate_b] + prompt = structured.ainvoke.await_args.args[0] + assert "complete-context" in prompt + assert "role validation is absent" in prompt + assert "SINGLETON-MUST-NOT-BE-SENT" not in prompt + + def test_same_anchor_independent_roots_are_not_candidates(self): + authorization = _real_issue( + line=25, + title="Request processing defect", + reason="The caller can update another workspace because ownership is never checked.", + code_snippet="service.update(request)", + ) + transaction = _real_issue( + line=25, + title="Request processing defect", + reason="The database transaction is committed before the downstream write completes.", + code_snippet="service.update(request)", + ) + + assert not issues_are_semantic_dedup_candidates( + authorization, + transaction, + ) + + @pytest.mark.asyncio(loop_scope="function") + async def test_cross_file_shared_root_keeps_both_locations(self): + first = _real_issue( + file="config/RagConfig.java", + line=30, + title="RAG constructor arity no longer matches shared configuration", + reason=( + "The shared RAG configuration now requires four constructor " + "arguments, but this caller supplies three and cannot compile." + ), + ) + second = _real_issue( + file="test/RagConfigTest.java", + line=70, + title="RAG constructor arity no longer matches shared configuration", + reason=( + "The shared RAG configuration requires four constructor arguments; " + "this test still supplies three and cannot compile." + ), + ) + assert issues_are_semantic_dedup_candidates(first, second) + llm = MagicMock() + structured = MagicMock() + structured.ainvoke = AsyncMock( + return_value=SemanticDeduplicationDecision(duplicate_groups=[ + SemanticDuplicateGroup( + keeper_index=0, + duplicate_indices=[1], + confidence="HIGH", + rationale="One constructor contract change breaks both callers.", + ) + ]) + ) + llm.with_structured_output.return_value = structured + + result = await deduplicate_final_issues_llm(llm, [first, second]) + + assert len(result) == 1 + assert result[0].file == "config/RagConfig.java" + assert result[0].relatedLocations == ["test/RagConfigTest.java:70"] + assert "Also affects: test/RagConfigTest.java:70" in result[0].reason @pytest.mark.asyncio(loop_scope="function") async def test_empty_returns_empty(self): diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_3_full.py b/python-ecosystem/inference-orchestrator/tests/test_stage_3_full.py index 903e94636..2c9afb152 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_3_full.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_3_full.py @@ -1,12 +1,15 @@ """Tests for stage_3_aggregation: summarizers, dismissed issues, MCP stage 3.""" import json import pytest +from types import SimpleNamespace from unittest.mock import MagicMock, AsyncMock, patch +from model.output_schemas import CodeReviewIssue from service.review.orchestrator.stage_3_aggregation import ( execute_stage_3_aggregation, _summarize_issues_for_stage_3, _summarize_plan_for_stage_3, _extract_dismissed_issues, + _stage_3_with_mcp, ) @@ -34,7 +37,7 @@ def test_severity_counts(self): assert "HIGH: 2" in result assert "MEDIUM: 1" in result - def test_top_findings_priority_order(self): + def test_complete_records_keep_stable_verification_identity(self): critical = MagicMock() critical.severity = "CRITICAL" critical.category = "SECURITY" @@ -52,12 +55,11 @@ def test_top_findings_priority_order(self): low.reason = "Naming convention" result = _summarize_issues_for_stage_3([low, critical]) - lines = result.split("\n") - # Critical should appear before LOW in the top findings section - top_lines = [l for l in lines if "[CRITICAL]" in l or "[LOW]" in l] - assert len(top_lines) == 2 - assert "CRITICAL" in top_lines[0] - assert "LOW" in top_lines[1] + records = json.loads(result.split("Complete verification records (JSON):\n", 1)[1]) + assert records[0]["verification_id"] == "issue_0" + assert records[0]["original_id"] == "l1" + assert records[1]["verification_id"] == "issue_1" + assert records[1]["original_id"] == "c1" def test_all_issue_ids_listed(self): issues = [] @@ -190,6 +192,9 @@ async def test_basic_no_mcp(self): request.changedFiles = ["a.py"] request.targetBranchName = "main" request.previousCodeAnalysisIssues = [] + request.currentCommitHash = None + request.commitHash = None + request.taskContext = None plan = MagicMock() plan.file_groups = [] @@ -220,6 +225,9 @@ async def test_incremental_review_context(self): request.changedFiles = [] request.targetBranchName = "" request.previousCodeAnalysisIssues = ["prev1", "prev2"] + request.currentCommitHash = None + request.commitHash = None + request.taskContext = None plan = MagicMock() plan.file_groups = [] @@ -237,7 +245,7 @@ async def test_incremental_review_context(self): @pytest.mark.asyncio(loop_scope="function") async def test_mcp_stage_dispatches(self): - """When use_mcp_tools=True and target_branch given, dispatches to MCP.""" + """An immutable reviewed commit enables the MCP verification loop.""" llm = MagicMock() mcp_client = MagicMock() @@ -249,6 +257,9 @@ async def test_mcp_stage_dispatches(self): request.changedFiles = [] request.targetBranchName = "main" request.previousCodeAnalysisIssues = [] + request.currentCommitHash = "abc123" + request.commitHash = None + request.taskContext = None plan = MagicMock() plan.file_groups = [] @@ -266,3 +277,64 @@ async def test_mcp_stage_dispatches(self): ) mock_mcp.assert_called_once() assert result["dismissed_issue_ids"] == ["x"] + + +class TestStage3McpVerification: + @pytest.mark.asyncio(loop_scope="function") + async def test_successful_revision_read_validates_fresh_issue_dismissal(self): + issue = CodeReviewIssue( + file="src/a.py", line=10, severity="HIGH", category="BUG_RISK", + reason="Claim to verify.", suggestedFixDescription="Fix it.", + ) + request = SimpleNamespace( + projectVcsWorkspace="workspace", + projectVcsRepoSlug="repo", + ) + tool_response = SimpleNamespace( + content="", + tool_calls=[{ + "id": "call-1", + "name": "getBranchFileContent", + "args": { + "filePath": "src/a.py", + "verificationId": "issue_0", + }, + }], + response_metadata={}, + ) + final_response = SimpleNamespace( + content=( + "Verified report\n" + '' + ), + tool_calls=[], + response_metadata={}, + ) + bound_llm = MagicMock() + bound_llm.ainvoke = AsyncMock( + side_effect=[tool_response, final_response] + ) + llm = MagicMock() + llm.bind_tools.return_value = bound_llm + mcp_client = MagicMock() + mcp_client.session.call_tool = AsyncMock(return_value=SimpleNamespace( + content=[SimpleNamespace(text="current source")] + )) + + result = await _stage_3_with_mcp( + llm, + request, + "prompt", + mcp_client, + "commit-abc", + {"issue_0": issue}, + ) + + assert result["report"] == "Verified report" + assert result["dismissed_issue_ids"] == [] + assert result["dismissed_issue_keys"] == ["issue_0"] + assert result["dismissed_issue_object_ids"] == [id(issue)] + call_args = mcp_client.session.call_tool.await_args.args[1] + assert call_args["branch"] == "commit-abc" + assert call_args["startLine"] == 1 + assert call_args["endLine"] == 90 diff --git a/python-ecosystem/inference-orchestrator/tests/test_stage_3_helpers.py b/python-ecosystem/inference-orchestrator/tests/test_stage_3_helpers.py index fece667b5..6424bc2de 100644 --- a/python-ecosystem/inference-orchestrator/tests/test_stage_3_helpers.py +++ b/python-ecosystem/inference-orchestrator/tests/test_stage_3_helpers.py @@ -10,6 +10,8 @@ _summarize_issues_for_stage_3, _summarize_plan_for_stage_3, _extract_dismissed_issues, + _stage_3_verification_issue_map, + _validated_mcp_dismissals, ) @@ -38,16 +40,17 @@ def test_with_issues(self): assert "HIGH" in result assert "ISS-1" in result - def test_sorting_by_severity(self): - issues = [ - CodeReviewIssue(id="L1", file="a.py", line=1, severity="LOW", category="CODE_QUALITY", reason="x", suggestedFixDescription="f"), - CodeReviewIssue(id="C1", file="b.py", line=2, severity="CRITICAL", category="BUG_RISK", reason="y", suggestedFixDescription="f"), - ] + def test_lists_every_issue_with_stable_verification_ids(self): + issues = [CodeReviewIssue( + id=f"I-{index}", file=f"src/f{index}.py", line=index + 2, + severity="LOW", category="CODE_QUALITY", reason=f"reason {index}", + suggestedFixDescription="fix", + ) for index in range(14)] result = _summarize_issues_for_stage_3(issues) - # CRITICAL should appear before LOW in top findings - crit_pos = result.find("C1") - low_pos = result.find("L1") - assert crit_pos < low_pos + + assert '"verification_id":"issue_0"' in result + assert '"verification_id":"issue_13"' in result + assert '"original_id":"I-13"' in result def test_excludes_resolved_history_records(self): resolved = CodeReviewIssue( @@ -61,6 +64,44 @@ def test_excludes_resolved_history_records(self): assert "No issues" in result assert "OLD-1" not in result + def test_reason_compaction_only_removes_exact_repetition(self): + issue = CodeReviewIssue( + file="a.py", line=1, severity="HIGH", category="BUG_RISK", + title="Repeated title", + reason=( + "Repeated title\n\nRoot cause.\n\nMore proof.\n\n" + "More proof.\n\nImpact: request fails." + ), + suggestedFixDescription="Fix it.", + ) + + records = json.loads( + _summarize_issues_for_stage_3([issue]).split( + "Complete verification records (JSON):\n", 1 + )[1] + ) + + reason = records[0]["reason"] + assert "Repeated title" not in reason + assert "Root cause." in reason + assert reason.count("More proof.") == 1 + assert "Impact: request fails." in reason + + def test_related_locations_are_recovered_from_persisted_reason(self): + issue = CodeReviewIssue( + file="a.py", line=1, severity="HIGH", category="BUG_RISK", + reason="Root cause.\n\nAlso affects: b.py:20, c.py:30", + suggestedFixDescription="Fix it.", + ) + + records = json.loads( + _summarize_issues_for_stage_3([issue]).split( + "Complete verification records (JSON):\n", 1 + )[1] + ) + + assert records[0]["related_locations"] == ["b.py:20", "c.py:30"] + # ── _summarize_plan_for_stage_3 ────────────────────────────── @@ -109,3 +150,107 @@ def test_non_list_marker(self): content = '' clean, dismissed = _extract_dismissed_issues(content) assert dismissed == [] + + +class TestValidatedMcpDismissals: + def test_requires_successful_read_at_exact_revision_for_every_location(self): + issue = CodeReviewIssue( + file="src/a.py", line=10, severity="HIGH", category="BUG_RISK", + title="Shared defect", reason="One root cause.", + suggestedFixDescription="Fix it.", + relatedLocations=["src/b.py:20"], + ) + issue_map = _stage_3_verification_issue_map([issue]) + executor = SimpleNamespace(call_log=[ + { + "tool": "getBranchFileContent", + "args": { + "filePath": "src/a.py", "branch": "abc123", + "verificationId": "issue_0", + }, + "success": True, + "evidence_valid": True, + "evidence_complete_file": True, + }, + ]) + + assert _validated_mcp_dismissals( + ["issue_0"], issue_map, executor, "abc123" + ) == [] + + executor.call_log.append({ + "tool": "getBranchFileContent", + "args": { + "filePath": "src/b.py", "branch": "abc123", + "verificationId": "issue_0", + }, + "success": True, + "evidence_valid": True, + "evidence_complete_file": False, + "evidence_start_line": 1, + "evidence_end_line": 100, + }) + assert _validated_mcp_dismissals( + ["issue_0"], issue_map, executor, "abc123" + ) == ["issue_0"] + + def test_wrong_revision_failed_or_unknown_dismissal_fails_open(self): + issue = CodeReviewIssue( + file="src/a.py", line=10, severity="HIGH", category="BUG_RISK", + reason="Concrete defect.", suggestedFixDescription="Fix it.", + ) + issue_map = _stage_3_verification_issue_map([issue]) + executor = SimpleNamespace(call_log=[ + { + "tool": "getBranchFileContent", + "args": { + "filePath": "src/a.py", "branch": "main", + "verificationId": "issue_0", + }, + "success": True, + "evidence_valid": True, + "evidence_complete_file": True, + }, + { + "tool": "getBranchFileContent", + "args": { + "filePath": "src/a.py", "branch": "abc123", + "verificationId": "issue_0", + }, + "success": False, + "evidence_valid": False, + }, + ]) + + assert _validated_mcp_dismissals( + ["issue_0", "issue_99"], issue_map, executor, "abc123" + ) == [] + + def test_window_must_cover_the_bound_issue_line(self): + issue = CodeReviewIssue( + file="src/a.py", line=500, severity="HIGH", category="BUG_RISK", + reason="Concrete defect.", suggestedFixDescription="Fix it.", + ) + issue_map = _stage_3_verification_issue_map([issue]) + executor = SimpleNamespace(call_log=[{ + "tool": "getBranchFileContent", + "args": { + "filePath": "src/a.py", "branch": "abc123", + "verificationId": "issue_0", + }, + "success": True, + "evidence_valid": True, + "evidence_complete_file": False, + "evidence_start_line": 1, + "evidence_end_line": 100, + }]) + + assert _validated_mcp_dismissals( + ["issue_0"], issue_map, executor, "abc123" + ) == [] + + executor.call_log[0]["evidence_start_line"] = 420 + executor.call_log[0]["evidence_end_line"] = 580 + assert _validated_mcp_dismissals( + ["issue_0"], issue_map, executor, "abc123" + ) == ["issue_0"] From 6e829b0972fbc9f18a0cb12e968111eaad164de6 Mon Sep 17 00:00:00 2001 From: rostislav Date: Mon, 10 Aug 2026 19:39:15 +0300 Subject: [PATCH 5/6] fix(rag): harden branch indexing and generation cleanup Add bounded archive extraction, case-safe branch aliases, preserve RAG settings through compatibility updates, and clean up all registered branch generations. --- deployment/.env.sample | 5 + deployment/docker-compose.prod.yml | 3 + deployment/docker-compose.yml | 3 + .../service/BranchArchiveService.java | 136 +++++++++++++++++- .../service/BranchArchiveServiceTest.java | 71 +++++++++ .../service/RagOperationsServiceImpl.java | 29 ++-- .../service/RagOperationsServiceImplTest.java | 54 +++++++ .../project/service/ProjectService.java | 39 ++++- .../service/ProjectServiceRagConfigTest.java | 79 ++++++++++ .../core/index_manager/manager.py | 5 +- .../rag-pipeline/tests/test_index_manager.py | 18 +++ 11 files changed, 419 insertions(+), 23 deletions(-) create mode 100644 java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/ProjectServiceRagConfigTest.java diff --git a/deployment/.env.sample b/deployment/.env.sample index e580a2cc8..2d14fa682 100644 --- a/deployment/.env.sample +++ b/deployment/.env.sample @@ -29,6 +29,11 @@ PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES=20971520 # The same bound prevents branch reconciliation from falling back to an # unbounded sequence of provider file requests when an archive is unavailable. VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD=25 +# Repository archives are rejected before indexing if any uncompressed member, +# total uncompressed content, or file count crosses these per-job bounds. +RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=268435456 +RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=4294967296 +RAG_ARCHIVE_MAX_ENTRIES=500000 # Testing-only: route normal analyses through full prompt capture instead of an # LLM. Empty project IDs means every project; a comma-separated list scopes it. diff --git a/deployment/docker-compose.prod.yml b/deployment/docker-compose.prod.yml index 831377abb..96f2c71da 100644 --- a/deployment/docker-compose.prod.yml +++ b/deployment/docker-compose.prod.yml @@ -167,6 +167,9 @@ services: PR_ENRICHMENT_MAX_FILE_SIZE_BYTES: ${PR_ENRICHMENT_MAX_FILE_SIZE_BYTES:-5242880} PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES: ${PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES:-20971520} CODECROW_VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD: ${VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD:-25} + RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES: ${RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES:-268435456} + RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES: ${RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES:-4294967296} + RAG_ARCHIVE_MAX_ENTRIES: ${RAG_ARCHIVE_MAX_ENTRIES:-500000} ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES:-15} ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES:-5} ANALYSIS_PROMPT_DRY_RUN_ENABLED: ${ANALYSIS_PROMPT_DRY_RUN_ENABLED:-false} diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index cbe8434e9..306e6891e 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -176,6 +176,9 @@ services: PR_ENRICHMENT_MAX_FILE_SIZE_BYTES: ${PR_ENRICHMENT_MAX_FILE_SIZE_BYTES:-5242880} PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES: ${PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES:-20971520} CODECROW_VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD: ${VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD:-25} + RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES: ${RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES:-268435456} + RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES: ${RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES:-4294967296} + RAG_ARCHIVE_MAX_ENTRIES: ${RAG_ARCHIVE_MAX_ENTRIES:-500000} ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES:-15} ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES:-5} ANALYSIS_PROMPT_DRY_RUN_ENABLED: ${ANALYSIS_PROMPT_DRY_RUN_ENABLED:-false} diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java index 802369aa6..01bca47b1 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java @@ -5,6 +5,7 @@ import org.rostilos.codecrow.vcsclient.VcsClientProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.*; @@ -29,17 +30,40 @@ public class BranchArchiveService { private static final Logger log = LoggerFactory.getLogger(BranchArchiveService.class); - /** - * Maximum single-file size retained by the in-memory extraction API. - * Directory extraction, used by RAG indexing, streams entries directly to - * disk and does not impose this limit on an archive or its members. - */ + /** Maximum single-file size retained by the in-memory extraction API. */ private static final long MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; + static final long DEFAULT_MAX_ARCHIVE_ENTRY_SIZE_BYTES = 256L * 1024 * 1024; + static final long DEFAULT_MAX_ARCHIVE_EXTRACTED_SIZE_BYTES = 4L * 1024 * 1024 * 1024; + static final int DEFAULT_MAX_ARCHIVE_ENTRIES = 500_000; private final VcsClientProvider vcsClientProvider; + private final long maxArchiveEntrySizeBytes; + private final long maxArchiveExtractedSizeBytes; + private final int maxArchiveEntries; + @Autowired public BranchArchiveService(VcsClientProvider vcsClientProvider) { + this( + vcsClientProvider, + envLong("RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES", DEFAULT_MAX_ARCHIVE_ENTRY_SIZE_BYTES), + envLong("RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES", DEFAULT_MAX_ARCHIVE_EXTRACTED_SIZE_BYTES), + envInt("RAG_ARCHIVE_MAX_ENTRIES", DEFAULT_MAX_ARCHIVE_ENTRIES)); + } + + BranchArchiveService( + VcsClientProvider vcsClientProvider, + long maxArchiveEntrySizeBytes, + long maxArchiveExtractedSizeBytes, + int maxArchiveEntries) { this.vcsClientProvider = vcsClientProvider; + if (maxArchiveEntrySizeBytes <= 0 + || maxArchiveExtractedSizeBytes <= 0 + || maxArchiveEntries <= 0) { + throw new IllegalArgumentException("Archive extraction limits must be positive"); + } + this.maxArchiveEntrySizeBytes = maxArchiveEntrySizeBytes; + this.maxArchiveExtractedSizeBytes = maxArchiveExtractedSizeBytes; + this.maxArchiveEntries = maxArchiveEntries; } /** @@ -223,6 +247,7 @@ private Set extractFilesFromArchive( int skippedLarge = 0; int skippedNotNeeded = 0; int skippedUnsafe = 0; + ArchiveExtractionBudget budget = new ArchiveExtractionBudget(); try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(archiveFile))) { ZipEntry entry; @@ -233,11 +258,13 @@ private Set extractFilesFromArchive( } String relativePath = stripArchiveRoot(entry.getName()); + ArchiveEntryBudget entryBudget = budget.beginEntry(relativePath, entry.getSize()); // Skip files we don't need if (neededFiles != null && !neededFiles.isEmpty() && !neededFiles.contains(relativePath)) { skippedNotNeeded++; + drainEntry(zis, entryBudget); zis.closeEntry(); continue; } @@ -247,7 +274,7 @@ private Set extractFilesFromArchive( // The in-memory API must not retain an unbounded archive // member. RAG uses extractFilesToDirectory below instead, // which streams every accepted entry to disk. - ZipEntryContent content = readZipEntryBounded(zis); + ZipEntryContent content = readZipEntryBounded(zis, entryBudget); if (content.tooLarge()) { log.debug("Skipping large file {} (more than {} bytes)", @@ -304,6 +331,7 @@ private Set extractFilesToDirectory( int skippedBinary = 0; int skippedNotNeeded = 0; int skippedUnsafe = 0; + ArchiveExtractionBudget budget = new ArchiveExtractionBudget(); try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(archiveFile))) { ZipEntry entry; @@ -314,9 +342,11 @@ private Set extractFilesToDirectory( } String relativePath = stripArchiveRoot(entry.getName()); + ArchiveEntryBudget entryBudget = budget.beginEntry(relativePath, entry.getSize()); if (neededFiles != null && !neededFiles.isEmpty() && !neededFiles.contains(relativePath)) { skippedNotNeeded++; + drainEntry(zis, entryBudget); zis.closeEntry(); continue; } @@ -325,13 +355,16 @@ private Set extractFilesToDirectory( Path targetFile = resolveTargetFile(targetDirectory, relativePath); if (targetFile == null) { skippedUnsafe++; + drainEntry(zis, entryBudget); zis.closeEntry(); continue; } byte[] prefix = zis.readNBytes(1024); + entryBudget.consume(prefix.length); if (isBinary(prefix, prefix.length)) { skippedBinary++; + drainEntry(zis, entryBudget); zis.closeEntry(); continue; } @@ -341,8 +374,12 @@ private Set extractFilesToDirectory( byte[] buffer = new byte[8192]; int length; while ((length = zis.read(buffer)) > 0) { + entryBudget.consume(length); output.write(buffer, 0, length); } + } catch (IOException extractionFailure) { + Files.deleteIfExists(targetFile); + throw extractionFailure; } setWorldReadable(targetFile, false); extractedFiles.add(relativePath); @@ -413,13 +450,17 @@ static String stripArchiveRoot(String entryPath) { return entryPath; } - private ZipEntryContent readZipEntryBounded(ZipInputStream zis) throws IOException { + private ZipEntryContent readZipEntryBounded( + ZipInputStream zis, + ArchiveEntryBudget entryBudget + ) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); byte[] buffer = new byte[8192]; long totalBytes = 0; boolean tooLarge = false; int len; while ((len = zis.read(buffer)) > 0) { + entryBudget.consume(len); totalBytes += len; if (totalBytes > MAX_FILE_SIZE_BYTES) { tooLarge = true; @@ -432,6 +473,32 @@ private ZipEntryContent readZipEntryBounded(ZipInputStream zis) throws IOExcepti : new ZipEntryContent(baos.toByteArray(), false); } + private void drainEntry(ZipInputStream zis, ArchiveEntryBudget entryBudget) throws IOException { + byte[] buffer = new byte[8192]; + int length; + while ((length = zis.read(buffer)) > 0) { + entryBudget.consume(length); + } + } + + private static long envLong(String name, long fallback) { + try { + long value = Long.parseLong(System.getenv().getOrDefault(name, String.valueOf(fallback))); + return value > 0 ? value : fallback; + } catch (NumberFormatException ignored) { + return fallback; + } + } + + private static int envInt(String name, int fallback) { + try { + int value = Integer.parseInt(System.getenv().getOrDefault(name, String.valueOf(fallback))); + return value > 0 ? value : fallback; + } catch (NumberFormatException ignored) { + return fallback; + } + } + /** * Checks if the data appears to be binary by looking for null bytes * in the first 1024 bytes. @@ -468,6 +535,61 @@ private record ZipEntryContent(byte[] bytes, boolean tooLarge) { private static final ZipEntryContent TOO_LARGE = new ZipEntryContent(null, true); } + private final class ArchiveExtractionBudget { + private long extractedBytes; + private int entries; + + private ArchiveEntryBudget beginEntry(String relativePath, long declaredSize) throws IOException { + entries++; + if (entries > maxArchiveEntries) { + throw new IOException("Repository archive exceeds RAG_ARCHIVE_MAX_ENTRIES=" + + maxArchiveEntries); + } + if (declaredSize > maxArchiveEntrySizeBytes) { + throw new IOException("Repository archive entry '" + relativePath + + "' exceeds RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=" + + maxArchiveEntrySizeBytes); + } + if (declaredSize >= 0 + && declaredSize > maxArchiveExtractedSizeBytes - extractedBytes) { + throw new IOException("Repository archive exceeds RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=" + + maxArchiveExtractedSizeBytes); + } + return new ArchiveEntryBudget(relativePath, this); + } + + private void consume(String relativePath, long entryBytes, int length) throws IOException { + if (length > maxArchiveEntrySizeBytes - entryBytes) { + throw new IOException("Repository archive entry '" + relativePath + + "' exceeds RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=" + + maxArchiveEntrySizeBytes); + } + if (length > maxArchiveExtractedSizeBytes - extractedBytes) { + throw new IOException("Repository archive exceeds RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=" + + maxArchiveExtractedSizeBytes); + } + extractedBytes += length; + } + } + + private static final class ArchiveEntryBudget { + private final String relativePath; + private final ArchiveExtractionBudget archiveBudget; + private long bytes; + + private ArchiveEntryBudget( + String relativePath, + ArchiveExtractionBudget archiveBudget) { + this.relativePath = relativePath; + this.archiveBudget = archiveBudget; + } + + private void consume(int length) throws IOException { + archiveBudget.consume(relativePath, bytes, length); + bytes += length; + } + } + public record ArchiveSnapshot( Map contents, Set presentFiles diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java index 00c1a03be..b01f99023 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveServiceTest.java @@ -217,6 +217,77 @@ void shouldStreamLargeArchiveEntriesToDiskWithoutRetainingThem(@TempDir Path tem assertThat(tempDir.resolve("repository/src/StillIndexed.java")).exists(); } + @Test + void extractToDirectory_rejectsOversizedEntryAndDeletesPartialFile(@TempDir Path tempDir) + throws Exception { + service = new BranchArchiveService(vcsClientProvider, 32, 1024, 10); + VcsConnection conn = new VcsConnection(); + when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); + byte[] zipBytes = createZip(Map.of( + "root/oversized.txt", "x".repeat(64) + )); + when(vcsClient.downloadRepositoryArchiveToFile( + anyString(), anyString(), anyString(), any(Path.class))) + .thenAnswer(inv -> { + Files.write(inv.getArgument(3, Path.class), zipBytes); + return (long) zipBytes.length; + }); + Path targetDirectory = tempDir.resolve("repository"); + + assertThatThrownBy(() -> service.downloadAndExtractFilesToDirectory( + conn, "ws", "repo", "main", null, targetDirectory)) + .isInstanceOf(IOException.class) + .hasMessageContaining("RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=32"); + assertThat(targetDirectory.resolve("oversized.txt")).doesNotExist(); + } + + @Test + void extractToDirectory_rejectsArchiveOverTotalExtractedLimit(@TempDir Path tempDir) + throws Exception { + service = new BranchArchiveService(vcsClientProvider, 64, 40, 10); + VcsConnection conn = new VcsConnection(); + when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); + byte[] zipBytes = createZip(Map.of( + "root/first.txt", "a".repeat(24), + "root/second.txt", "b".repeat(24) + )); + when(vcsClient.downloadRepositoryArchiveToFile( + anyString(), anyString(), anyString(), any(Path.class))) + .thenAnswer(inv -> { + Files.write(inv.getArgument(3, Path.class), zipBytes); + return (long) zipBytes.length; + }); + + assertThatThrownBy(() -> service.downloadAndExtractFilesToDirectory( + conn, "ws", "repo", "main", null, tempDir.resolve("repository"))) + .isInstanceOf(IOException.class) + .hasMessageContaining("RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=40"); + } + + @Test + void extractionCountsUnrequestedEntriesAgainstArchiveEntryLimit(@TempDir Path tempDir) + throws Exception { + service = new BranchArchiveService(vcsClientProvider, 64, 1024, 1); + VcsConnection conn = new VcsConnection(); + when(vcsClientProvider.getClient(conn)).thenReturn(vcsClient); + byte[] zipBytes = createZip(Map.of( + "root/requested.txt", "requested", + "root/unrequested.txt", "unrequested" + )); + when(vcsClient.downloadRepositoryArchiveToFile( + anyString(), anyString(), anyString(), any(Path.class))) + .thenAnswer(inv -> { + Files.write(inv.getArgument(3, Path.class), zipBytes); + return (long) zipBytes.length; + }); + + assertThatThrownBy(() -> service.downloadAndExtractFilesToDirectory( + conn, "ws", "repo", "main", Set.of("missing.txt"), + tempDir.resolve("repository"))) + .isInstanceOf(IOException.class) + .hasMessageContaining("RAG_ARCHIVE_MAX_ENTRIES=1"); + } + @Test void snapshotReportsBinaryPathPresenceWithoutLoadingItsContent(@TempDir Path tempDir) throws Exception { diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java index a6a53ea8a..e37f2e845 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java @@ -31,6 +31,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -838,13 +839,15 @@ public boolean deleteBranchIndex( "message", String.format("Deleting RAG index for branch '%s'", branchName))); boolean success; - Optional exactIndex = ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branchName) - .filter(index -> index.getActiveGeneration() != null); - if (exactIndex.isPresent() && branchGenerationRepository != null) { - List generations = branchGenerationRepository - .findByBranchIndexIdOrderByCreatedAtDesc(exactIndex.get().getId()); - success = !generations.isEmpty(); + Optional trackedIndex = ragBranchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branchName); + List generations = trackedIndex.isPresent() + && branchGenerationRepository != null + ? branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc( + trackedIndex.get().getId()) + : List.of(); + if (!generations.isEmpty()) { + success = true; for (RagBranchIndexGeneration generation : generations) { success &= ragPipelineClient.deleteBranch( project.getWorkspace().getName(), project.getNamespace(), @@ -900,8 +903,13 @@ public Map cleanupStaleBranches( String baseBranch = getBaseBranch(project); try { - // Get indexed branches - List indexedBranches = ragPipelineClient.getIndexedBranches(workspaceSlug, projectSlug); + // The durable registry is authoritative for exact-generation + // branches. Merge it with legacy shared-collection discovery so + // stale cleanup remains compatible with both storage models. + Set indexedBranches = new LinkedHashSet<>( + ragBranchIndexRepository.findBranchNamesByProjectId(project.getId())); + indexedBranches.addAll( + ragPipelineClient.getIndexedBranches(workspaceSlug, projectSlug)); // Determine branches to keep: base branch + active branches Set branchesToKeep = new HashSet<>(activeBranches); @@ -933,9 +941,8 @@ public Map cleanupStaleBranches( for (String branch : staleBranches) { try { - boolean success = ragPipelineClient.deleteBranch(workspaceSlug, projectSlug, branch); + boolean success = deleteBranchIndex(project, branch, eventConsumer); if (success) { - ragBranchIndexRepository.deleteByProjectIdAndBranchName(project.getId(), branch); deletedBranches.add(branch); } else { failedBranches.add(branch); diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java index 417fc0336..cbec29237 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java @@ -13,10 +13,13 @@ import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.project.config.RagConfig; import org.rostilos.codecrow.core.model.rag.RagBranchIndex; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; import org.rostilos.codecrow.core.model.workspace.Workspace; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; import org.rostilos.codecrow.core.service.AnalysisJobService; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; @@ -53,6 +56,9 @@ class RagOperationsServiceImplTest { @Mock private RagBranchIndexRepository ragBranchIndexRepository; + @Mock + private RagBranchIndexGenerationRepository branchGenerationRepository; + @Mock private VcsClientProvider vcsClientProvider; @@ -72,6 +78,8 @@ void setUp() { ragBranchIndexRepository, vcsClientProvider, ragPipelineClient); + ReflectionTestUtils.setField( + service, "branchGenerationRepository", branchGenerationRepository); testProject = new Project(); ReflectionTestUtils.setField(testProject, "id", 100L); @@ -485,6 +493,52 @@ void testCleanupStaleBranches_DeletesStaleBranch() throws Exception { assertThat(result).containsEntry("total_deleted", 1); } + @Test + void cleanupStaleBranchesDeletesEveryRegisteredExactGeneration() throws Exception { + setupRagEnabled(); + setupVcsBinding(); + setupProjectWithWorkspaceAndNamespace(); + when(ragBranchIndexRepository.findBranchNamesByProjectId(100L)) + .thenReturn(List.of("main", "stale-exact")); + when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) + .thenReturn(List.of("main")); + + RagBranchIndex branchIndex = new RagBranchIndex( + testProject, "stale-exact", RagBranchIndexKind.DURABLE); + branchIndex.setId(501L); + RagBranchIndexGeneration first = new RagBranchIndexGeneration(); + first.setCollectionName("cc_generation_1"); + RagBranchIndexGeneration second = new RagBranchIndexGeneration(); + second.setCollectionName("cc_generation_2"); + when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "stale-exact")) + .thenReturn(Optional.of(branchIndex)); + when(branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc(501L)) + .thenReturn(List.of(first, second)); + when(ragPipelineClient.deleteBranch( + "test-ws", "test-ns", "stale-exact", "cc_generation_1")) + .thenReturn(true); + when(ragPipelineClient.deleteBranch( + "test-ws", "test-ns", "stale-exact", "cc_generation_2")) + .thenReturn(true); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = mock(Consumer.class); + + Map result = service.cleanupStaleBranches( + testProject, Set.of(), eventConsumer); + + assertThat(result).containsEntry("status", "success"); + assertThat(result).containsEntry("total_deleted", 1); + assertThat(result.get("deleted_branches")).isEqualTo(List.of("stale-exact")); + verify(ragPipelineClient).deleteBranch( + "test-ws", "test-ns", "stale-exact", "cc_generation_1"); + verify(ragPipelineClient).deleteBranch( + "test-ws", "test-ns", "stale-exact", "cc_generation_2"); + verify(ragBranchIndexRepository) + .deleteByProjectIdAndBranchName(100L, "stale-exact"); + verify(ragPipelineClient, never()) + .deleteBranch("my-workspace", "my-repo", "stale-exact"); + } + @Test void testUpdateBranchIndex_WhenNotEnabled() { ReflectionTestUtils.setField(service, "ragApiEnabled", false); diff --git a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java index 31f4d689f..e6fa2c4df 100644 --- a/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java +++ b/java-ecosystem/services/web-server/src/main/java/org/rostilos/codecrow/webserver/project/service/ProjectService.java @@ -745,7 +745,21 @@ public Project updateRagConfig( Boolean transientBranchIndexesEnabled) { Project project = projectRepository.findByWorkspaceIdAndId(workspaceId, projectId) .orElseThrow(() -> new NoSuchElementException("Project not found")); + return updateRagConfig(project, enabled, branch, includePatterns, excludePatterns, + multiBranchEnabled, branchRetentionDays, indexedBranches, + transientBranchIndexesEnabled); + } + private Project updateRagConfig( + Project project, + boolean enabled, + String branch, + java.util.List includePatterns, + java.util.List excludePatterns, + Boolean multiBranchEnabled, + Integer branchRetentionDays, + java.util.List indexedBranches, + Boolean transientBranchIndexesEnabled) { ProjectConfig currentConfig = project.getConfiguration(); boolean useLocalMcp = currentConfig != null && currentConfig.useLocalMcp(); boolean useMcpTools = currentConfig != null && currentConfig.useMcpTools(); @@ -781,8 +795,13 @@ public Project updateRagConfig( java.util.List excludePatterns, Boolean multiBranchEnabled, Integer branchRetentionDays) { - return updateRagConfig(workspaceId, projectId, enabled, branch, includePatterns, excludePatterns, - multiBranchEnabled, branchRetentionDays, null, null); + Project project = projectRepository.findByWorkspaceIdAndId(workspaceId, projectId) + .orElseThrow(() -> new NoSuchElementException("Project not found")); + RagConfig currentRag = currentRagConfig(project); + return updateRagConfig(project, enabled, branch, includePatterns, excludePatterns, + multiBranchEnabled, branchRetentionDays, + currentRag != null ? currentRag.indexedBranches() : null, + currentRag != null ? currentRag.transientBranchIndexesEnabled() : null); } /** @@ -796,8 +815,20 @@ public Project updateRagConfig( String branch, java.util.List includePatterns, java.util.List excludePatterns) { - return updateRagConfig(workspaceId, projectId, enabled, branch, includePatterns, excludePatterns, - null, null, null, null); + Project project = projectRepository.findByWorkspaceIdAndId(workspaceId, projectId) + .orElseThrow(() -> new NoSuchElementException("Project not found")); + RagConfig currentRag = currentRagConfig(project); + return updateRagConfig(project, enabled, branch, includePatterns, excludePatterns, + currentRag != null ? currentRag.multiBranchEnabled() : null, + currentRag != null ? currentRag.branchRetentionDays() : null, + currentRag != null ? currentRag.indexedBranches() : null, + currentRag != null ? currentRag.transientBranchIndexesEnabled() : null); + } + + private RagConfig currentRagConfig(Project project) { + return project.getConfiguration() != null + ? project.getConfiguration().ragConfig() + : null; } @Transactional diff --git a/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/ProjectServiceRagConfigTest.java b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/ProjectServiceRagConfigTest.java new file mode 100644 index 000000000..e0458d8f7 --- /dev/null +++ b/java-ecosystem/services/web-server/src/test/java/org/rostilos/codecrow/webserver/project/service/ProjectServiceRagConfigTest.java @@ -0,0 +1,79 @@ +package org.rostilos.codecrow.webserver.project.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; + +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ProjectServiceRagConfigTest { + + @Mock + private ProjectRepository projectRepository; + + @InjectMocks + private ProjectService projectService; + + private Project project; + + @BeforeEach + void setUp() { + project = new Project(); + project.setConfiguration(new ProjectConfig( + false, + "main", + null, + new RagConfig( + true, + "main", + List.of("src/**"), + List.of("target/**"), + true, + 30, + List.of("develop", "release"), + true))); + when(projectRepository.findByWorkspaceIdAndId(10L, 20L)) + .thenReturn(Optional.of(project)); + when(projectRepository.save(project)).thenReturn(project); + } + + @Test + void eightArgumentCompatibilityUpdatePreservesNewerBranchSettings() { + Project updated = projectService.updateRagConfig( + 10L, 20L, true, "main", + List.of("app/**"), List.of("build/**"), false, 14); + + RagConfig rag = updated.getConfiguration().ragConfig(); + assertThat(rag.multiBranchEnabled()).isFalse(); + assertThat(rag.branchRetentionDays()).isEqualTo(14); + assertThat(rag.indexedBranches()).containsExactly("develop", "release"); + assertThat(rag.transientBranchIndexesEnabled()).isTrue(); + } + + @Test + void sixArgumentCompatibilityUpdatePreservesAllMultiBranchSettings() { + Project updated = projectService.updateRagConfig( + 10L, 20L, false, "develop", + List.of("service/**"), List.of("generated/**")); + + RagConfig rag = updated.getConfiguration().ragConfig(); + assertThat(rag.enabled()).isFalse(); + assertThat(rag.branch()).isEqualTo("develop"); + assertThat(rag.multiBranchEnabled()).isTrue(); + assertThat(rag.branchRetentionDays()).isEqualTo(30); + assertThat(rag.indexedBranches()).containsExactly("develop", "release"); + assertThat(rag.transientBranchIndexesEnabled()).isTrue(); + } +} diff --git a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py index ee9358009..ffce9bdc7 100644 --- a/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py +++ b/python-ecosystem/rag-pipeline/src/rag_pipeline/core/index_manager/manager.py @@ -262,7 +262,10 @@ def _get_branch_operator_alias( readable = re.sub(r"[^a-zA-Z0-9_-]", "_", raw_branch).lower() if not readable: readable = "branch" - if readable != raw_branch.lower() or len(readable) > 64: + # Branch identities are case-sensitive even though readable aliases are + # normalized to lowercase. Hash whenever normalization changes the + # original identity so case-only branches cannot share an alias. + if readable != raw_branch or len(readable) > 64: readable = ( f"{readable[:64]}_" f"{hashlib.sha256(raw_branch.encode('utf-8')).hexdigest()[:10]}" diff --git a/python-ecosystem/rag-pipeline/tests/test_index_manager.py b/python-ecosystem/rag-pipeline/tests/test_index_manager.py index d8ad3b9c1..aba24d6d5 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_manager.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_manager.py @@ -428,6 +428,24 @@ def test_branch_operator_alias_is_readable_and_branch_specific(self): "Workspace", "Project", "release/1.2" ).startswith("rag_workspace__project__release_1_2_") + def test_branch_operator_alias_preserves_case_sensitive_identity(self): + from rag_pipeline.core.index_manager.manager import RAGIndexManager + + manager = object.__new__(RAGIndexManager) + manager.config = MagicMock(qdrant_collection_prefix="rag") + manager.config.qdrant_collection_prefix = "rag" + + lowercase = manager._get_branch_operator_alias( + "Workspace", "Project", "feature" + ) + uppercase = manager._get_branch_operator_alias( + "Workspace", "Project", "Feature" + ) + + assert lowercase != uppercase + assert lowercase == "rag_workspace__project__feature" + assert uppercase.startswith("rag_workspace__project__feature_") + def test_readable_alias_publication_requires_immutable_generation_target(self): from rag_pipeline.core.index_manager.manager import RAGIndexManager From 6bd206ed455059bb94e67401f9aeaa1c9fd28345 Mon Sep 17 00:00:00 2001 From: rostislav Date: Tue, 11 Aug 2026 01:08:33 +0300 Subject: [PATCH 6/6] frontend submodule version bump --- frontend | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend b/frontend index baa2853c6..cc7c44a3c 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit baa2853c665ba99b00861501ad1f5f984b4dd20a +Subproject commit cc7c44a3c0623f3e9259e5f388a27bdea0527100