Fix macro classification for Copilot-only MCP tools

ts/packages/copilot-macros/README.md

@@ -56,7 +56,8 @@ The public contracts in `src/contracts.ts` include:
 
 Each macro and step has an execution class:
 
-- `replayable`: TypeAgent can inspect and call the named MCP tool.
+- `replayable`: TypeAgent's replay host inspected and found the named MCP tool
+  when the draft was created.
 - `agentRequired`: the step needs Copilot's live tool and permission surface.
 
 Execution class is selected for the whole macro. A run never replays a prefix
@@ -72,8 +73,21 @@ before handing later steps to the agent.
 - infer the result type and up to 50 result paths as postconditions.
 
 It does not infer arbitrary semantic parameters, loops, branches, or a
-generalized program from one example. Calls with an MCP server name are
-classified as replayable. Calls without one are classified as agent-required.
+generalized program from one example. Induction is asynchronous: calls with an
+MCP server name are inspected through the supplied replay host using the
+recorded working directory. Only tools available to that host are classified
+as replayable. Native calls, missing servers or tools, and calls without a
+configured replay host are classified as agent-required. The captured server
+and tool names are preserved for the agent runner.
+
+An MCP server name alone does not establish replayability. For example,
+`github-mcp-server/web_search` may be available in Copilot but not in
+TypeAgent. Such a call produces an agent-required draft with a review warning,
+not an unapprovable replayable draft. Connection, authentication, and tool-list
+errors still fail draft creation; they are not treated as absent capabilities.
+Unreadable configuration files and invalid entries for the requested server
+also fail discovery rather than silently selecting the agent runner.
+Inspection does not invoke the recorded tools.
 
 ## Validation and approval
 
@@ -86,6 +100,15 @@ records its schema fingerprint. Approval then writes the next version with
 state `approved`. Disabling an approved macro also writes a new version.
 Agent-guided adaptations are saved as separate drafts.
 
+Classification does not silently change during approval or execution. A
+replayable tool disappearing after draft creation still blocks approval, and
+existing approved replayable macros retain their preflight checks. To correct
+an older draft that misclassified a Copilot-only MCP tool, create a new draft
+from its original trace, review it, and explicitly approve it. Existing
+versions are not rewritten.
+Tool inspection refreshes the connected server's advertised catalog, so draft
+creation cannot leave approval or replay checking a stale tool schema.
+
 ## Deterministic replay
 
 Replay preflights the complete macro before invoking step one:
@@ -110,6 +133,11 @@ If a macro is `agentRequired`, or the caller selects the `agent` preference,
 `MacroManager.runMacro()` returns an `AgentRunnerLaunchPayload`. The payload
 contains the approved macro, supplied inputs, handoff reason, execution
 budgets, and candidate provenance. The package does not launch the runner.
+The runner inherits available Copilot tools, including MCP tools outside
+TypeAgent's replay host, subject to live permissions. Its instructions restrict
+tool use to the approved procedure, inspection, required discovery, and
+successful candidate submission. An unavailable or denied tool stops the run;
+the runner does not install tools or change permissions.
 
 After a successful agent-guided run, `submitMacroCandidate()` can save an
 adapted procedure. It verifies handoff identity, source version, execution

ts/packages/copilot-macros/src/contracts.ts

@@ -7,6 +7,8 @@ export interface RecordedToolCall {
     mcpServerName?: string;
     arguments?: unknown;
     result?: unknown;
+    // Parsed model-facing tool text, distinct from the raw event/UI result.
+    modelResult?: unknown;
     status: "completed" | "failed" | "denied";
     permission?: unknown;
 }
@@ -328,6 +330,7 @@ export interface ReplayToolContext {
 }
 
 export interface ReplayToolHost {
+    // Missing servers/tools return undefined; connection and inspection failures throw.
     inspectTool(
         mcpServerName: string | undefined,
         toolName: string,

ts/packages/copilot-macros/src/macroDefinition.ts

@@ -11,14 +11,21 @@ import type {
     MacroValidationIssue,
     MacroValidationReport,
     RecordedInteractionTrace,
+    ReplayToolHost,
     ValueExpression,
 } from "./contracts.js";
 
-function classifyTool(
-    _toolName: string,
+async function classifyTool(
+    toolName: string,
     mcpServerName: string | undefined,
-): MacroExecutionClass {
-    return mcpServerName ? "replayable" : "agentRequired";
+    cwd: string,
+    replayHost: ReplayToolHost | undefined,
+): Promise<MacroExecutionClass> {
+    if (!mcpServerName || !replayHost) return "agentRequired";
+    const descriptor = await replayHost.inspectTool(mcpServerName, toolName, {
+        cwd,
+    });
+    return descriptor ? "replayable" : "agentRequired";
 }
 
 function getValueType(value: unknown): MacroValueType {
@@ -151,25 +158,48 @@ function convertArguments(
         : { kind: "template", value, bindings };
 }
 
-export function induceMacroFromTrace(
+export async function induceMacroFromTrace(
     traceId: string,
     trace: RecordedInteractionTrace,
     macroId: string,
     name: string,
     description: string,
     createdAt: string,
-): CopilotToolMacro {
+    replayHost?: ReplayToolHost,
+): Promise<CopilotToolMacro> {
     const warnings: string[] = [];
     const inputs: MacroInput[] = [];
     const steps: MacroStep[] = [];
-    trace.toolCalls.forEach((call, index) => {
+    const executionClasses: MacroExecutionClass[] = [];
+    for (const call of trace.toolCalls) {
+        executionClasses.push(
+            await classifyTool(
+                call.name,
+                call.mcpServerName,
+                trace.cwd,
+                replayHost,
+            ),
+        );
+    }
+    const agentRequired = executionClasses.includes("agentRequired");
+    const calls = trace.toolCalls.map((call) =>
+        agentRequired && call.modelResult !== undefined
+            ? { ...call, result: call.modelResult }
+            : call,
+    );
+    for (const [index, call] of calls.entries()) {
         const id = `step-${index + 1}`;
-        const executionClass = classifyTool(call.name, call.mcpServerName);
+        const executionClass = executionClasses[index];
         if (executionClass === "agentRequired") {
             warnings.push(
                 `${id} uses ${call.mcpServerName ? `${call.mcpServerName}/` : ""}${call.name} and requires agent-guided execution.`,
             );
         }
+        if (agentRequired && call.modelResult === undefined) {
+            warnings.push(
+                `${id} has no captured model-facing result. Review its result guards and recapture if the runner cannot observe the required fields.`,
+            );
+        }
         if (call.status !== "completed") {
             warnings.push(
                 `${id} was captured with status ${call.status} and requires review.`,
@@ -186,7 +216,7 @@ export function induceMacroFromTrace(
                 id,
                 trace.prompt,
                 steps,
-                trace.toolCalls.slice(0, index),
+                calls.slice(0, index),
                 inputs,
                 warnings,
             ),
@@ -196,7 +226,7 @@ export function induceMacroFromTrace(
                 ? {}
                 : { postconditions: inferPostconditions(call.result) }),
         });
-    });
+    }
 
     return {
         schemaVersion: 1,
@@ -205,11 +235,7 @@ export function induceMacroFromTrace(
         name,
         description,
         state: "draft",
-        executionClass: steps.every(
-            (step) => step.executionClass === "replayable",
-        )
-            ? "replayable"
-            : "agentRequired",
+        executionClass: agentRequired ? "agentRequired" : "replayable",
         inputs,
         steps,
         sourceTraceId: traceId,

ts/packages/copilot-macros/src/macroManager.ts

@@ -227,13 +227,14 @@ export class MacroManager {
         if (!request.name.trim()) throw new Error("Macro name is required.");
         return this.mutateCatalog(async () => {
             const trace = await this.readTrace(request.traceId);
-            const macro = induceMacroFromTrace(
+            const macro = await induceMacroFromTrace(
                 request.traceId,
                 trace,
                 randomUUID(),
                 request.name.trim(),
                 request.description?.trim() ?? trace.prompt,
                 new Date().toISOString(),
+                this.replayHost,
             );
             await this.writeVersion(macro);
             await this.upsertSummary(macro);