Finish what indexing starts: live project switching, scoped embeddings, persistent selection

packages/api/src/__tests__/embeddings-route.test.ts

@@ -0,0 +1,141 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { scheduleEmbeddingPass, getEmbeddingPassState } = vi.hoisted(() => ({
+  scheduleEmbeddingPass: vi.fn(),
+  getEmbeddingPassState: vi.fn(),
+}));
+
+vi.mock('@codegraph/core', () => ({
+  codeGraphService: {
+    resolveProjectRootPath: vi.fn(),
+  },
+  getGraphClient: vi.fn(),
+  indexProject: Object.assign(vi.fn(), {
+    scheduleEmbeddingPass,
+    getEmbeddingPassState,
+  }),
+}));
+
+import { codeGraphService, getGraphClient } from '@codegraph/core';
+import { statsRoutes } from '../routes/stats';
+
+const mockedResolveProjectRootPath = vi.mocked(codeGraphService.resolveProjectRootPath);
+const mockedGetGraphClient = vi.mocked(getGraphClient);
+
+const idlePass = {
+  running: false,
+  scope: null,
+  startedAt: null,
+};
+
+function graphClientWith(rows: Array<{ label: string; total: number; withEmbedding: number }>) {
+  return {
+    roQuery: vi.fn().mockResolvedValue({ data: rows, metadata: [] }),
+  };
+}
+
+describe('embedding routes', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    getEmbeddingPassState.mockReturnValue(idlePass);
+    scheduleEmbeddingPass.mockResolvedValue({
+      embedded: 2,
+      skipped: 1,
+      errors: 0,
+      durationMs: 250,
+      byType: { File: 2 },
+    });
+  });
+
+  it('reports global coverage with an explicit global scope', async () => {
+    const client = graphClientWith([{ label: 'File', total: 4, withEmbedding: 3 }]);
+    mockedGetGraphClient.mockResolvedValue(client as never);
+
+    const response = await statsRoutes.request('/api/embeddings/status');
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({
+      scope: { type: 'global' },
+      embeddingPass: idlePass,
+      labels: [{ label: 'File', total: 4, withEmbedding: 3, coverage: 75 }],
+    });
+    expect(client.roQuery).toHaveBeenCalledWith(
+      expect.not.stringContaining('$projectPath'),
+      { params: {} },
+    );
+    expect(getEmbeddingPassState).toHaveBeenCalledWith(undefined);
+  });
+
+  it('resolves projectId and scopes coverage with an exact-or-slash-prefix boundary', async () => {
+    mockedResolveProjectRootPath.mockResolvedValue('/repos/app/');
+    const client = graphClientWith([{ label: 'Function', total: 2, withEmbedding: 1 }]);
+    mockedGetGraphClient.mockResolvedValue(client as never);
+
+    const response = await statsRoutes.request('/api/embeddings/status?projectId=project-1');
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({
+      scope: { type: 'project', projectId: 'project-1', rootPath: '/repos/app' },
+      embeddingPass: idlePass,
+      labels: [{ label: 'Function', total: 2, withEmbedding: 1, coverage: 50 }],
+    });
+    const [cypher, options] = client.roQuery.mock.calls[0]!;
+    expect(cypher).toContain('n.filePath = $projectPath OR n.filePath STARTS WITH $projectPathPrefix');
+    expect(options).toEqual({
+      params: { projectPath: '/repos/app', projectPathPrefix: '/repos/app/' },
+    });
+    expect(getEmbeddingPassState).toHaveBeenCalledWith('project-1');
+  });
+
+  it('returns 404 for an unknown status projectId before querying coverage', async () => {
+    mockedResolveProjectRootPath.mockResolvedValue(undefined);
+
+    const response = await statsRoutes.request('/api/embeddings/status?projectId=missing');
+
+    expect(response.status).toBe(404);
+    expect(await response.json()).toEqual({ error: 'Project not found.' });
+    expect(mockedGetGraphClient).not.toHaveBeenCalled();
+  });
+
+  it('generates only for the resolved project scope', async () => {
+    mockedResolveProjectRootPath.mockResolvedValue('/repos/app/');
+    const client = graphClientWith([]);
+    mockedGetGraphClient.mockResolvedValue(client as never);
+
+    const response = await statsRoutes.request('/api/embeddings/generate', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ projectId: 'project-1' }),
+    });
+
+    expect(response.status).toBe(200);
+    expect(scheduleEmbeddingPass).toHaveBeenCalledWith({
+      client,
+      force: false,
+      projectId: 'project-1',
+      rootPath: '/repos/app',
+    });
+    expect(await response.json()).toEqual({
+      scope: { type: 'project', projectId: 'project-1', rootPath: '/repos/app' },
+      embedded: 2,
+      skipped: 1,
+      errors: 0,
+      durationMs: 250,
+      byType: { File: 2 },
+      message: 'Embedded 2 nodes in 0.3s (1 skipped, 0 errors)',
+    });
+  });
+
+  it('rejects a non-string projectId before graph access', async () => {
+    const response = await statsRoutes.request('/api/embeddings/generate', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ projectId: 42 }),
+    });
+
+    expect(response.status).toBe(400);
+    expect(await response.json()).toEqual({ error: 'projectId must be a non-empty string.' });
+    expect(mockedGetGraphClient).not.toHaveBeenCalled();
+    expect(scheduleEmbeddingPass).not.toHaveBeenCalled();
+  });
+});

packages/api/src/routes/stats.ts

@@ -1,9 +1,50 @@
 import { Hono } from 'hono';
-import { codeGraphService, knowledgeService, getGraphClient, embedAllNodes } from '@codegraph/core';
+import { codeGraphService, knowledgeService, getGraphClient, indexProject } from '@codegraph/core';
+import type { GraphClient } from '@codegraph/graph';
 import { safeErrorMessage } from '../safe-error';
 
 export const statsRoutes = new Hono();
 
+type EmbeddingScope =
+  | { type: 'global' }
+  | { type: 'project'; projectId: string; rootPath: string };
+
+interface EmbeddingPassState {
+  running: boolean;
+  scope: EmbeddingScope | null;
+  startedAt: string | null;
+}
+
+interface EmbeddingGenerateResult {
+  embedded: number;
+  skipped: number;
+  errors: number;
+  durationMs: number;
+  byType: Record<string, number>;
+}
+
+const embeddingCoordinator = indexProject as typeof indexProject & {
+  getEmbeddingPassState(projectId?: string): EmbeddingPassState;
+  scheduleEmbeddingPass(options: {
+    client: GraphClient;
+    force: boolean;
+    projectId?: string;
+    rootPath?: string;
+  }): Promise<EmbeddingGenerateResult>;
+};
+
+function normalizeProjectRoot(rootPath: string): string {
+  return rootPath.replace(/\/+$/, '') || '/';
+}
+
+async function resolveEmbeddingScope(projectId: string | undefined): Promise<EmbeddingScope | null> {
+  if (projectId === undefined) return { type: 'global' };
+  const rootPath = await codeGraphService.resolveProjectRootPath(projectId);
+  return rootPath === undefined
+    ? null
+    : { type: 'project', projectId, rootPath: normalizeProjectRoot(rootPath) };
+}
+
 /** GET /api/projects — list indexed projects */
 statsRoutes.get('/api/projects', async (c) => {
   try {
@@ -63,7 +104,20 @@ statsRoutes.get('/api/knowledge/stats', async (c) => {
 /** GET /api/embeddings/status — embedding coverage per label */
 statsRoutes.get('/api/embeddings/status', async (c) => {
   try {
+    const projectId = c.req.query('projectId') || undefined;
+    const scope = await resolveEmbeddingScope(projectId);
+    if (scope === null) return c.json({ error: 'Project not found.' }, 404);
+
     const client = await getGraphClient();
+    const projectFilter = scope.type === 'project'
+      ? 'AND (n.filePath = $projectPath OR n.filePath STARTS WITH $projectPathPrefix)'
+      : '';
+    const params = scope.type === 'project'
+      ? {
+          projectPath: scope.rootPath,
+          projectPathPrefix: scope.rootPath === '/' ? '/' : `${scope.rootPath}/`,
+        }
+      : {};
 
     // Get counts of nodes with and without embeddings per label
     const result = await client.roQuery<{
@@ -73,11 +127,13 @@ statsRoutes.get('/api/embeddings/status', async (c) => {
     }>(
       `MATCH (n)
        WHERE labels(n)[0] IS NOT NULL
+         ${projectFilter}
        WITH labels(n)[0] AS label, n
        RETURN label,
               count(n) AS total,
               sum(CASE WHEN n.embedding IS NOT NULL THEN 1 ELSE 0 END) AS withEmbedding
        ORDER BY total DESC`,
+      { params },
     );
 
     const labels = result.data.map((row) => ({
@@ -87,7 +143,11 @@ statsRoutes.get('/api/embeddings/status', async (c) => {
       coverage: row.total > 0 ? Math.round((row.withEmbedding / row.total) * 100) : 0,
     }));
 
-    return c.json({ labels });
+    return c.json({
+      scope,
+      embeddingPass: embeddingCoordinator.getEmbeddingPassState(projectId),
+      labels,
+    });
   } catch (error) {
     return c.json({ error: safeErrorMessage('GET /api/embeddings/status', error, 'Failed to fetch embedding status.') }, 500);
   }
@@ -97,11 +157,30 @@ statsRoutes.get('/api/embeddings/status', async (c) => {
 statsRoutes.post('/api/embeddings/generate', async (c) => {
   try {
     const body = await c.req.json().catch(() => ({}));
-    const force = (body as Record<string, unknown>).force === true;
+    const values = body as Record<string, unknown>;
+    const rawProjectId = values.projectId;
+    if (
+      rawProjectId !== undefined &&
+      (typeof rawProjectId !== 'string' || rawProjectId.trim().length === 0)
+    ) {
+      return c.json({ error: 'projectId must be a non-empty string.' }, 400);
+    }
+    const projectId = typeof rawProjectId === 'string' ? rawProjectId : undefined;
+    const scope = await resolveEmbeddingScope(projectId);
+    if (scope === null) return c.json({ error: 'Project not found.' }, 404);
+    const force = values.force === true;
+    const client = await getGraphClient();
 
-    const result = await embedAllNodes({ force });
+    const result = await embeddingCoordinator.scheduleEmbeddingPass({
+      client,
+      force,
+      ...(scope.type === 'project'
+        ? { projectId: scope.projectId, rootPath: scope.rootPath }
+        : {}),
+    });
 
     return c.json({
+      scope,
       ...result,
       message: `Embedded ${result.embedded} nodes in ${(result.durationMs / 1000).toFixed(1)}s (${result.skipped} skipped, ${result.errors} errors)`,
     });

packages/core/src/__tests__/indexer-embedding-continuation.integration.test.ts

@@ -0,0 +1,111 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createClient, resolveEmbeddedBinaryPaths, type GraphClient } from '@codegraph/graph';
+import { generateEmbeddings } from '@codegraph/plugin-nlp';
+import { getEmbeddingPassState, scheduleEmbeddingPass } from '../embed-pass';
+import { indexProject } from '../indexer';
+
+vi.mock('@codegraph/plugin-nlp', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@codegraph/plugin-nlp')>();
+  return {
+    ...actual,
+    isEmbeddingAvailable: () => true,
+    generateEmbeddings: vi.fn(async (texts: string[]) => {
+      await new Promise((resolve) => setTimeout(resolve, 20));
+      return { embeddings: texts.map(() => Array.from({ length: 384 }, () => 0.1)) };
+    }),
+  };
+});
+
+const describeIfAvailable = resolveEmbeddedBinaryPaths() ? describe : describe.skip;
+
+describeIfAvailable('post-index embedding continuation', () => {
+  let client: GraphClient;
+  let dataDir: string;
+  let projectDir: string;
+  let previousEmbeddingProvider: string | undefined;
+
+  beforeAll(async () => {
+    previousEmbeddingProvider = process.env['CODEGRAPH_EMBEDDING_PROVIDER'];
+    process.env['CODEGRAPH_EMBEDDING_PROVIDER'] = 'local';
+    dataDir = await mkdtemp('/tmp/cge-');
+    client = await createClient({
+      driver: 'falkordblite',
+      databasePath: dataDir,
+      graphName: 'embedding_continuation',
+    } as never);
+
+    projectDir = mkdtempSync('/tmp/cgp-');
+    writeFileSync(
+      join(projectDir, 'fixture.ts'),
+      [
+        'export const answer = 42;',
+        '',
+        'export function readAnswer(): number {',
+        '  return answer;',
+        '}',
+        '',
+      ].join('\n'),
+    );
+  }, 60_000);
+
+  afterAll(async () => {
+    await client?.close();
+    if (dataDir) await rm(dataDir, { recursive: true, force: true });
+    if (projectDir) rmSync(projectDir, { recursive: true, force: true });
+    if (previousEmbeddingProvider === undefined) {
+      delete process.env['CODEGRAPH_EMBEDDING_PROVIDER'];
+    } else {
+      process.env['CODEGRAPH_EMBEDDING_PROVIDER'] = previousEmbeddingProvider;
+    }
+  });
+
+  it('schedules the remaining project nodes and deduplicates concurrent manual generation', async () => {
+    vi.mocked(generateEmbeddings).mockClear();
+
+    const indexed = await indexProject(projectDir, {
+      client,
+      includePatterns: ['*.ts'],
+      deferEmbeddings: true,
+      gitSync: false,
+      force: true,
+    });
+
+    expect(indexed.success, indexed.errorMessages.join('\n')).toBe(true);
+    expect(getEmbeddingPassState(indexed.projectId)).toMatchObject({
+      running: true,
+      scope: { type: 'project', projectId: indexed.projectId, rootPath: projectDir },
+    });
+
+    await scheduleEmbeddingPass({
+      client,
+      projectId: indexed.projectId,
+      rootPath: projectDir,
+      force: false,
+    });
+
+    const coverage = await client.roQuery<{ total: number; withEmbedding: number }>(
+      `MATCH (n)
+       WHERE n.projectId = $projectId AND
+             (n:File OR n:Function OR n:Class OR n:Interface OR n:Variable OR n:Type OR n:Component)
+       RETURN count(n) AS total,
+              sum(CASE WHEN n.embedding IS NOT NULL THEN 1 ELSE 0 END) AS withEmbedding`,
+      { params: { projectId: indexed.projectId } },
+    );
+
+    expect(coverage.data).toEqual([{ total: 3, withEmbedding: 3 }]);
+    expect(
+      vi.mocked(generateEmbeddings).mock.calls.reduce(
+        (total, [texts]) => total + (texts as string[]).length,
+        0,
+      ),
+    ).toBe(3);
+    expect(getEmbeddingPassState(indexed.projectId)).toEqual({
+      running: false,
+      scope: null,
+      startedAt: null,
+    });
+  }, 60_000);
+});