Improve indexing and search for imported how-to docs

ts/packages/agents/browser/src/agent/browserMemoryService.mts

@@ -12,6 +12,7 @@ import type {
     MemoryServiceCapabilities,
     MemoryService,
     MemorySource,
+    PersonalHowToService,
 } from "@typeagent/memory-service";
 import { waitForMemoryJob } from "@typeagent/memory-service/rpc";
 
@@ -90,18 +91,21 @@ export interface BrowserSourceKnowledge {
     relationships: MemoryKnowledgeGraph["relationships"];
 }
 
-export interface BrowserSourceKnowledge {
-    source: MemorySource;
-    entities: MemoryKnowledgeGraph["entities"];
-    topics: MemoryKnowledgeGraph["topics"];
-    relationships: MemoryKnowledgeGraph["relationships"];
+export interface BrowserIngestResult extends BrowserSourceKnowledge {
+    warnings: string[];
+    howTo?: {
+        enabled: boolean;
+        candidateCount: number;
+    };
 }
 
 export class BrowserMemoryService {
     private corpusIdPromise: Promise<string> | undefined;
     private graphVersion = 0;
 
-    public constructor(private readonly client: MemoryService) {}
+    public constructor(
+        private readonly client: MemoryService & Partial<PersonalHowToService>,
+    ) {}
 
     public async ingest(
         document: BrowserMemoryDocument,
@@ -110,8 +114,9 @@ export class BrowserMemoryService {
             signal?: AbortSignal;
             onProgress?: (progress: JobProgress) => void;
             maxCharsPerChunk?: number;
+            reportHowToStatus?: boolean;
         } = {},
-    ): Promise<BrowserSourceKnowledge> {
+    ): Promise<BrowserIngestResult> {
         const corpusId = await this.getCorpusId();
         const result = await this.client.ingestDocument({
             corpusId,
@@ -160,7 +165,52 @@ export class BrowserMemoryService {
                 `Memory ingestion completed but source '${document.url}' was not found`,
             );
         }
-        return knowledge;
+        const warnings = [...job.warnings];
+        if (!options.reportHowToStatus) {
+            return { ...knowledge, warnings };
+        }
+        try {
+            if (
+                !this.client.getPersonalHowToSettings ||
+                !this.client.listProcedureCandidates
+            ) {
+                throw new Error("Personal how-to service is not available");
+            }
+            const settings =
+                await this.client.getPersonalHowToSettings(corpusId);
+            if (!settings.enabled || !settings.detectCandidates) {
+                return {
+                    ...knowledge,
+                    warnings,
+                    howTo: { enabled: false, candidateCount: 0 },
+                };
+            }
+            const candidates = await this.client.listProcedureCandidates(
+                corpusId,
+                ["detected", "draft"],
+            );
+            return {
+                ...knowledge,
+                warnings,
+                howTo: {
+                    enabled: true,
+                    candidateCount: candidates.filter((candidate) =>
+                        candidate.citations.some(
+                            (citation) =>
+                                citation.sourceId === result.sourceId &&
+                                citation.revisionId === result.revisionId,
+                        ),
+                    ).length,
+                },
+            };
+        } catch (error) {
+            warnings.push(
+                `Could not check how-to candidates: ${
+                    error instanceof Error ? error.message : String(error)
+                }`,
+            );
+            return { ...knowledge, warnings };
+        }
     }
 
     public async search(
@@ -495,7 +545,7 @@ export class BrowserMemoryService {
 }
 
 export function getBrowserMemoryService(
-    client: MemoryService,
+    client: MemoryService & Partial<PersonalHowToService>,
 ): BrowserMemoryService {
     let service = adapters.get(client);
     if (service === undefined) {

ts/packages/agents/browser/src/agent/knowledge/actions/indexingActions.mts

@@ -19,12 +19,16 @@ export async function indexWebPageContent(
         mode?: "basic" | "content" | "full";
         extractedKnowledge?: any;
         activityType?: "visited" | "captured";
+        reportHowToStatus?: boolean;
     },
     context: SessionContext<BrowserActionContext>,
 ): Promise<{
     indexed: boolean;
     knowledgeExtracted: boolean;
     entityCount: number;
+    warnings?: string[];
+    howTo?: { enabled: boolean; candidateCount: number };
+    error?: string;
 }> {
     try {
         if (parameters.extractedKnowledge) {
@@ -45,6 +49,9 @@ export async function indexWebPageContent(
         const combinedTextContent = extractionInputs
             .map((input) => `## ${input.title}\n\n${input.textContent}`)
             .join("\n\n");
+        if (!combinedTextContent) {
+            throw new Error("The page did not contain enough text to index");
+        }
 
         const memoryService = context.agentContext.browserMemoryService;
         if (memoryService === undefined) {
@@ -60,20 +67,27 @@ export async function indexWebPageContent(
                 activityType: parameters.activityType ?? "captured",
             },
             parameters.mode ?? "content",
+            parameters.reportHowToStatus === undefined
+                ? {}
+                : { reportHowToStatus: parameters.reportHowToStatus },
         );
         debug(`Stored current page in durable memory: ${parameters.url}`);
 
         return {
             indexed: true,
             knowledgeExtracted: parameters.extractKnowledge,
             entityCount: knowledge.entities.length,
+            ...(parameters.reportHowToStatus
+                ? { warnings: knowledge.warnings, howTo: knowledge.howTo }
+                : {}),
         };
     } catch (error) {
         console.error("Error indexing page content:", error);
         return {
             indexed: false,
             knowledgeExtracted: false,
             entityCount: 0,
+            error: error instanceof Error ? error.message : String(error),
         };
     }
 }

ts/packages/agents/browser/test/browserMemoryService.test.ts

@@ -135,6 +135,13 @@ function createClient(): jest.Mocked<MemoryServiceClient> {
             },
             warnings: [],
         })),
+        getPersonalHowToSettings: jest.fn(async () => ({
+            revision: 0,
+            updatedAt: "2026-01-01T00:00:00.000Z",
+            enabled: true,
+            detectCandidates: true,
+        })),
+        listProcedureCandidates: jest.fn(async () => []),
         waitForJob: jest.fn(async () => ({
             jobId: "job-1",
             corpusId: "browser-corpus",
@@ -151,6 +158,96 @@ function createClient(): jest.Mocked<MemoryServiceClient> {
 }
 
 describe("BrowserMemoryService", () => {
+    test("reports candidates from the ingested page revision and job warnings", async () => {
+        const client = createClient();
+        client.getJob.mockResolvedValue({
+            ...(await client.getJob("job-1")),
+            warnings: ["Extractor needs attention"],
+        });
+        client.listProcedureCandidates.mockResolvedValue([
+            {
+                candidateId: "page",
+                corpusId: "browser-corpus",
+                state: "detected",
+                title: "How to save",
+                steps: ["Capture", "Review"],
+                citations: [{ sourceId: "source-1", revisionId: "revision-1" }],
+                createdAt: "2026-01-01T00:00:00.000Z",
+                updatedAt: "2026-01-01T00:00:00.000Z",
+            },
+            {
+                candidateId: "other",
+                corpusId: "browser-corpus",
+                state: "detected",
+                title: "Older page revision",
+                steps: ["One", "Two"],
+                citations: [{ sourceId: "source-1", revisionId: "revision-0" }],
+                createdAt: "2026-01-01T00:00:00.000Z",
+                updatedAt: "2026-01-01T00:00:00.000Z",
+            },
+        ]);
+
+        const result = await new BrowserMemoryService(client).ingest(
+            {
+                url: "https://example.test/page",
+                title: "How to save",
+                markdown: "## Steps\n1. Capture\n2. Review",
+            },
+            "content",
+            { reportHowToStatus: true },
+        );
+
+        expect(result.howTo).toEqual({ enabled: true, candidateCount: 1 });
+        expect(result.warnings).toEqual(["Extractor needs attention"]);
+        expect(client.listProcedureCandidates).toHaveBeenCalledWith(
+            "browser-corpus",
+            ["detected", "draft"],
+        );
+    });
+
+    test("reports disabled how-to detection without listing candidates", async () => {
+        const client = createClient();
+        client.getPersonalHowToSettings.mockResolvedValue({
+            revision: 1,
+            updatedAt: "2026-01-01T00:00:00.000Z",
+            enabled: true,
+            detectCandidates: false,
+        });
+
+        const result = await new BrowserMemoryService(client).ingest(
+            {
+                url: "https://example.test/page",
+                title: "Page",
+                markdown: "# Page",
+            },
+            "content",
+            { reportHowToStatus: true },
+        );
+        expect(result.howTo).toEqual({ enabled: false, candidateCount: 0 });
+        expect(client.listProcedureCandidates).not.toHaveBeenCalled();
+    });
+
+    test("preserves the saved page and warns when candidate status cannot be read", async () => {
+        const client = createClient();
+        client.listProcedureCandidates.mockRejectedValue(
+            new Error("Candidate store unavailable"),
+        );
+        const result = await new BrowserMemoryService(client).ingest(
+            {
+                url: "https://example.test/page",
+                title: "Page",
+                markdown: "## Steps\n1. First\n2. Second",
+            },
+            "content",
+            { reportHowToStatus: true },
+        );
+        expect(result.source.sourceId).toBeDefined();
+        expect(result.howTo).toBeUndefined();
+        expect(result.warnings).toEqual([
+            "Could not check how-to candidates: Candidate store unavailable",
+        ]);
+    });
+
     test("shares an adapter for browser sessions using the same client", () => {
         const client = createClient();
 

ts/packages/agents/browserControlRpc/src/serviceTypes.ts

@@ -559,7 +559,16 @@ export type BrowserAgentInvokeFunctions = {
         textOnly?: boolean;
         mode?: string;
         extractedKnowledge?: any;
-    }): Promise<any>;
+        activityType?: "visited" | "captured";
+        reportHowToStatus?: boolean;
+    }): Promise<{
+        indexed: boolean;
+        knowledgeExtracted: boolean;
+        entityCount: number;
+        warnings?: string[];
+        howTo?: { enabled: boolean; candidateCount: number };
+        error?: string;
+    }>;
 
     checkPageIndexStatus(params: { url: string }): Promise<any>;
     getPageIndexedKnowledge(params: { url: string }): Promise<any>;

ts/packages/agents/browserExtension/src/extension/serviceWorker/contextMenu.ts

@@ -7,6 +7,7 @@ import {
     awaitConversationOps,
 } from "./dispatcherConnection";
 import { awaitCommand } from "@typeagent/dispatcher-types";
+import { indexPageContent } from "./messageHandlers";
 
 // RPC send function — set after RPC server is created in index.ts
 let rpcSendFn: ((name: string, ...args: any[]) => void) | undefined;
@@ -107,6 +108,38 @@ async function openChatAndStartMacroAuthoring(tabId: number): Promise<void> {
     }, 500);
 }
 
+async function savePage(tab: chrome.tabs.Tab): Promise<void> {
+    if (tab.id === undefined || !tab.url || !/^https?:\/\//i.test(tab.url)) {
+        console.error("Cannot save a tab without an HTTP(S) URL");
+        return;
+    }
+    const result = await indexPageContent(tab, true, {
+        activityType: "captured",
+        mode: "content",
+        reportHowToStatus: true,
+    });
+    if (
+        result.indexed &&
+        (result.warnings?.length || result.howTo === undefined)
+    ) {
+        await chrome.action.setBadgeText({ tabId: tab.id, text: "!" });
+        await chrome.action.setBadgeBackgroundColor({
+            tabId: tab.id,
+            color: "#d97706",
+        });
+    }
+    const status = !result.indexed
+        ? `Could not save page: ${result.error ?? "Unknown error"}`
+        : result.warnings?.length
+          ? `Page saved, but extraction needs attention: ${result.warnings.join("; ")}. See jobs in Memory Center.`
+          : result.howTo === undefined
+            ? "Page saved, but how-to status is unavailable. See jobs in Memory Center."
+            : !result.howTo.enabled
+              ? "Page saved. How-to detection is disabled for the browser corpus."
+              : `Page saved. ${result.howTo.candidateCount} how-to candidate(s). Open Memory Center and select TypeAgent Browser Memory to review.`;
+    await chrome.action.setTitle({ tabId: tab.id, title: status });
+}
+
 /**
  * Initializes the context menu items
  */
@@ -128,6 +161,12 @@ export function initializeContextMenu(): void {
         documentUrlPatterns: ["http://*/*", "https://*/*"],
     });
 
+    chrome.contextMenus.create({
+        title: "Save this page",
+        id: "saveThisPage",
+        documentUrlPatterns: ["http://*/*", "https://*/*"],
+    });
+
     chrome.contextMenus.create({
         type: "separator",
         id: "menuSeparator2",
@@ -238,6 +277,11 @@ export async function handleContextMenuClick(
             break;
         }
 
+        case "saveThisPage": {
+            await savePage(tab);
+            break;
+        }
+
         case "showWebsiteLibrary": {
             const knowledgeLibraryUrl = chrome.runtime.getURL(
                 "views/knowledgeLibrary.html",