fix(ai): make the editor tools discoverable and state what the user is on

.eslintignore

@@ -25,3 +25,6 @@ test/**/node_modules/**/*.js
 test/virtual-server-test.js
 test/spec/ESLintExtensionTest-files
 
+
+# AI panel model-test fixtures: deliberately broken pages the tests operate on, not code
+src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/fixtures/**

CLAUDE.md

@@ -48,6 +48,9 @@ Use `exec_js` to run JS in the Phoenix browser runtime. jQuery `$()` is global.
 
 **Check logs:** `get_browser_console_logs` with `filter` regex (e.g. `"AI UI"`, `"error"`) and `tail` — includes both browser console and Node.js (PhNode) logs. Use `get_terminal_logs` for Electron process output (only available if Phoenix was launched via `start_phoenix`).
 
+## AI model tests (behavioural tests of the AI panel)
+When asked to "run the AI test suite" / "run the model tests" / "run EC-1 and UB-2": call `run_ai_test_suite` (phoenix-builder MCP) with `suite` (`quick` | `all` | a suite name), or `tests` for specific IDs, or `resumeRunId` to continue. It installs the fixture, opens a run record, and returns the briefing plus the test documents from `src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/`. You are the runner and the judge — follow them exactly, deterministic checks first. After **every** test call `ai_test_progress` and tell the user one progress line. If the user says stop: `ai_test_progress({ runId, stop: true })`, then save. Finish with `save_ai_test_report`, then `compare_ai_test_reports({})`, and tell the user the report path, PASS/FAIL counts, any regressions, and the Observations section.
+
 ## Writing Tests
 - **Never use `awaits(number)`** (fixed-time waits) in tests — they cause flaky failures. Always use `awaitsFor(condition)` to wait for a specific condition to become true.
 - Use `editor.*` APIs (e.g. `editor.document.getText()`, `editor.getCursorPos()`, `editor.setSelection()`) instead of accessing `editor._codeMirror` directly.

phoenix-builder-mcp/README.md

@@ -97,6 +97,18 @@ Reloads the Phoenix app. Prompts to save unsaved files before reloading.
 ### `force_reload_phoenix`
 Force-reloads the Phoenix app without saving unsaved changes.
 
+### `run_ai_test_suite`
+Starts (or resumes) the AI panel model tests and hands the session everything it needs: installs the fixture project, gathers git revisions / CLI version / connected instance, opens a run record, and returns the runner briefing plus the test documents from `src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/`. The session is the runner and the judge. `suite`: `quick` (default), `all`, or a suite name; or `tests: ["EC-1","UB-2"]` for specific tests; or `resumeRunId` to continue a stopped run. Ask Claude: *"run the AI test suite"*, *"run just the plan-mode tests"*, *"run EC-1 and UB-2"*.
+
+### `ai_test_progress`
+Called by the runner after every test to record the result; also answers *"how far along is it?"* (`{ runId }`), lists runs (`{}`), and stops a run (`{ runId, stop: true }`). Progress lives in `reports/runs/<runId>.json`, which you can open at any time.
+
+### `save_ai_test_report`
+Writes the finished report to `reports/latest.md` inside the suite folder, overwriting the previous run (git history keeps earlier runs; `baseline.md` is never touched). A stopped run is saved with a Partial section listing the unrun tests.
+
+### `compare_ai_test_reports`
+Diffs two reports test by test — by default `latest.md` against `baseline.md`, or `against: "previous"` for the last committed run — and flags regressions, quality drops, and slower runs using the thresholds in `model_tests.md`.
+
 ## Typical Claude Code workflow
 
 ```

src-node/claude-code-agent.js

@@ -315,6 +315,43 @@ function _formatAnswers(answer) {
     return answerText.trim();
 }
 
+/**
+ * Render the editor context the panel sent into the line prepended to the
+ * prompt. The panel assembles it because that is where the data and the
+ * user's context chips already live; this only turns it into prose. Returns
+ * "" when the panel sent nothing, i.e. the user dismissed those chips.
+ */
+function _buildEditorContextLine(ctx) {
+    if (!ctx || (!ctx.activeFile && !ctx.livePreviewFile)) {
+        return "";
+    }
+    const parts = ["Editor state (auto-supplied, no tool call needed):"];
+    if (ctx.activeFile) {
+        parts.push("the user is editing " + ctx.activeFile + ".");
+        if (ctx.unsaved) {
+            // Read and Edit are buffer-safe here (the agent flushes the buffer
+            // first); only Grep sees stale disk. Saying "stale on disk" without that
+            // steered the model off Edit onto the editor API — no edit card, no undo.
+            parts.push("Unsaved (Read and Edit see the unsaved text as normal; Grep does not, so " +
+                "use searchEditorBuffers to search these): " + ctx.unsaved + ".");
+        }
+    }
+    if (ctx.livePreviewFile) {
+        parts.push(ctx.livePreviewFile === ctx.activeFile
+            ? "The live preview is showing that same file."
+            : "The live preview is showing " + ctx.livePreviewFile + ".");
+    }
+    // Say plainly when the lists are complete. Left merely to infer it, the
+    // agent calls getEditorState to check — the exact lookup this line is
+    // here to save.
+    parts.push(ctx.truncated
+        ? "Trust this over searching for it yourself; call getEditorState for the names cut " +
+          "from a list, or if you need the cursor, the selection or a fresher view."
+        : "That is the complete set. Trust it over searching or double-checking; call " +
+          "getEditorState only if you need the cursor, the selection or a fresher view.");
+    return parts.join(" ");
+}
+
 /**
  * Detect whether a PostToolUse `tool_response` represents an error result.
  * Used to suppress diff-card painting when the SDK's native Edit/Write itself
@@ -855,7 +892,8 @@ exports.checkAvailability = async function (opts) {
  *   aiProgress, aiTextStream, aiToolEdit, aiError, aiComplete
  */
 exports.sendPrompt = async function (params) {
-    const { prompt, projectPath, sessionAction, model, locale, selectionContext, images, envOverrides, permissionMode, additionalDirectories } = params;
+    const { prompt, projectPath, sessionAction, model, locale, selectionContext, editorContext,
+        images, envOverrides, permissionMode, additionalDirectories } = params;
     const requestId = Date.now().toString(36) + Math.random().toString(36).slice(2, 7);
 
     // Handle session
@@ -874,6 +912,12 @@ exports.sendPrompt = async function (params) {
 
     currentAbortController = new AbortController();
 
+    // Prepend what the user is looking at. The panel knows the active file,
+    // the unsaved buffers and the live preview target for certain, so stating
+    // them costs one line and removes the reason to go hunting: without it
+    // the model opens by grepping the project for a file already on screen.
+    const editorContextLine = _buildEditorContextLine(editorContext);
+
     // Prepend selection context to the prompt if available
     let enrichedPrompt = prompt;
     if (selectionContext) {
@@ -896,6 +940,9 @@ exports.sendPrompt = async function (params) {
                 " to read the selected content if needed." + previewSnippet + "\n" + prompt;
         }
     }
+    if (editorContextLine) {
+        enrichedPrompt = editorContextLine + "\n\n" + enrichedPrompt;
+    }
 
     // Run the query asynchronously — don't await here so we return requestId immediately
     _runQuery(requestId, enrichedPrompt, projectPath, model, currentAbortController.signal, locale, images, envOverrides, permissionMode, additionalDirectories)
@@ -1378,6 +1425,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale,
             "WebFetch", "WebSearch",
             "EnterPlanMode", "ExitPlanMode",
             "mcp__phoenix-editor__getEditorState",
+            "mcp__phoenix-editor__searchEditorBuffers",
             "mcp__phoenix-editor__takeScreenshot",
             "mcp__phoenix-editor__execJsInLivePreview",
             "mcp__phoenix-editor__execJsInEditor",
@@ -1399,6 +1447,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale,
                     " files to answer questions. Do not modify files.",
                 tools: ["Read", "Glob", "Grep",
                     "mcp__phoenix-editor__getEditorState",
+                    "mcp__phoenix-editor__searchEditorBuffers",
                     "mcp__phoenix-editor__takeScreenshot",
                     "mcp__phoenix-editor__execJsInLivePreview",
                     "mcp__phoenix-editor__editorDocs"]
@@ -1411,6 +1460,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale,
                     " only for new files.",
                 tools: ["Read", "Edit", "Write", "Glob", "Grep",
                     "mcp__phoenix-editor__getEditorState",
+                    "mcp__phoenix-editor__searchEditorBuffers",
                     "mcp__phoenix-editor__takeScreenshot",
                     "mcp__phoenix-editor__execJsInLivePreview",
                     "mcp__phoenix-editor__execJsInEditor",

src-node/mcp-editor-tools.js

@@ -54,7 +54,8 @@ const EXEC_PEER_TIMEOUT_MS = {
     getEditorState: 5000,
     takeScreenshot: 15000,
     controlEditor: 5000,
-    resizeLivePreview: 5000
+    resizeLivePreview: 5000,
+    searchEditorBuffers: 3000
 };
 
 // Floor for caller-provided timeouts (e.g. execJsInLivePreview's
@@ -154,7 +155,57 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
             }
             return _maybeAppendHint(result, hasClarification);
         },
-        { annotations: { readOnlyHint: true } }
+        {
+            annotations: { readOnlyHint: true },
+            alwaysLoad: true,
+            searchHint: "which file the user has open in Phoenix Code editor, plus cursor, selection, and what the live preview (an embedded browser rendering their HTML or Markdown) is showing"
+        }
+    );
+
+    const searchEditorBuffersTool = sdkModule.tool(
+        "searchEditorBuffers",
+        "Regex search over the UNSAVED open files only — the ones the editor-state line at the top of " +
+        "the prompt lists as unsaved. Those are the only files where Grep is wrong: Grep reads disk, and " +
+        "disk is stale for a buffer the user has edited but not saved. Use Grep for everything else; it " +
+        "is faster and covers the whole project. Only call this when the editor-state line names unsaved " +
+        "files. Returns matches {file, line, text}, searchedFiles (what this actually covered) and truncated.",
+        {
+            pattern: z.string().describe("Regex (default) or literal text to find"),
+            isRegex: z.boolean().optional().describe("false to match the pattern literally. Default true"),
+            caseSensitive: z.boolean().optional().describe("Default false"),
+            fileGlob: z.string().optional().describe("Limit to matching files, e.g. *.css"),
+            maxResults: z.number().optional().describe("Cap on matches returned. Default 50, max 200")
+        },
+        async function (args) {
+            let result;
+            try {
+                const found = await _execPeerWithTimeout(nodeConnector, "searchEditorBuffers",
+                    args || {}, "searchEditorBuffers");
+                let text;
+                if (found && found.error) {
+                    text = JSON.stringify(found);
+                } else if (!found || !found.searchedFiles || !found.searchedFiles.length) {
+                    text = "No unsaved files, so nothing in the editor differs from disk. Use Grep — " +
+                        "it is authoritative for the whole project right now.";
+                } else {
+                    text = JSON.stringify(found) +
+                        "\n\nThis searched ONLY the unsaved files in searchedFiles. Every other file " +
+                        "matches disk — use Grep for the rest of the project.";
+                }
+                result = { content: [{ type: "text", text: text }] };
+            } catch (err) {
+                result = {
+                    content: [{ type: "text", text: "Error searching unsaved files: " + err.message }],
+                    isError: true
+                };
+            }
+            return _maybeAppendHint(result, hasClarification);
+        },
+        {
+            annotations: { readOnlyHint: true },
+            alwaysLoad: true,
+            searchHint: "search the unsaved editor buffers, where Grep would see stale disk content"
+        }
     );
 
     const takeScreenshotTool = sdkModule.tool(
@@ -219,7 +270,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
             }
             return _maybeAppendHint(toolResult, hasClarification);
         },
-        { annotations: { readOnlyHint: true } }
+        {
+            annotations: { readOnlyHint: true },
+            alwaysLoad: true,
+            searchHint: "screenshot the user's Phoenix Code editor app window, or the page rendered in their live preview browser"
+        }
     );
 
     const execJsInLivePreviewTool = sdkModule.tool(
@@ -265,7 +320,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
             }
             return _maybeAppendHint(toolResult, hasClarification);
         },
-        { annotations: { readOnlyHint: true } }
+        {
+            annotations: { readOnlyHint: true },
+            alwaysLoad: true,
+            searchHint: "run JS in the user's live preview browser to inspect the rendered page's DOM, console or JS state"
+        }
     );
 
     const controlEditorTool = sdkModule.tool(
@@ -330,7 +389,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
             };
             return _maybeAppendHint(toolResult, hasClarification);
         },
-        { annotations: { readOnlyHint: true } }
+        {
+            annotations: { readOnlyHint: true },
+            alwaysLoad: true,
+            searchHint: "open, close or switch files in Phoenix Code, toggle the live preview browser"
+        }
     );
 
     const resizeLivePreviewTool = sdkModule.tool(
@@ -364,7 +427,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
             }
             return _maybeAppendHint(toolResult, hasClarification);
         },
-        { annotations: { readOnlyHint: true } }
+        {
+            annotations: { readOnlyHint: true },
+            alwaysLoad: true,
+            searchHint: "resize the user's live preview browser viewport to check a responsive layout"
+        }
     );
 
     const waitTool = sdkModule.tool(
@@ -383,7 +450,10 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
             };
             return _maybeAppendHint(toolResult, hasClarification);
         },
-        { annotations: { readOnlyHint: true } }
+        {
+            annotations: { readOnlyHint: true },
+            searchHint: "pause before re-checking the user's live preview browser"
+        }
     );
 
     const execJsInEditorTool = sdkModule.tool(
@@ -457,6 +527,9 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
                 };
             }
             return _maybeAppendHint(toolResult, hasClarification);
+        },
+        {
+            searchHint: "run JS against Phoenix Code's own editor API, not the page in its live preview"
         }
     );
 
@@ -527,6 +600,9 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
                 };
             }
             return _maybeAppendHint(toolResult, hasClarification);
+        },
+        {
+            searchHint: "read or change the user's Phoenix Code editor preferences"
         }
     );
 
@@ -574,7 +650,10 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
             };
             return _maybeAppendHint(toolResult, hasClarification);
         },
-        { annotations: { readOnlyHint: true } }
+        {
+            annotations: { readOnlyHint: true },
+            searchHint: "look up Phoenix Code editor feature or API documentation"
+        }
     );
 
     const getUserClarificationTool = sdkModule.tool(
@@ -607,12 +686,15 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors)
                 content: [{ type: "text", text: "No clarification queued." }]
             };
         },
-        { annotations: { readOnlyHint: true } }
+        {
+            annotations: { readOnlyHint: true },
+            searchHint: "read a follow-up the user typed into this conversation while you were still working"
+        }
     );
 
     return sdkModule.createSdkMcpServer({
         name: "phoenix-editor",
-        tools: [getEditorStateTool, takeScreenshotTool, execJsInLivePreviewTool,
+        tools: [getEditorStateTool, searchEditorBuffersTool, takeScreenshotTool, execJsInLivePreviewTool,
             execJsInEditorTool, editorPreferencesTool, editorDocsTool,
             controlEditorTool, resizeLivePreviewTool, waitTool, getUserClarificationTool]
     });

src/nls/root/strings.js

@@ -2659,6 +2659,8 @@ define({
     "AI_CHAT_TOOL_SCREENSHOT_LIVE_PREVIEW": "live preview",
     "AI_CHAT_TOOL_SCREENSHOT_FULL_EDITOR": "the full editor",
     "AI_CHAT_TOOL_LIVE_PREVIEW_JS": "Inspecting preview",
+    "AI_CHAT_TOOL_SEARCH_UNSAVED": "Search unsaved files",
+    "AI_CHAT_TOOL_SEARCH_UNSAVED_FOR": "Search unsaved files: {0}",
     "AI_CHAT_TOOL_EDITOR_JS": "Inspecting editor",
     "AI_CHAT_TOOL_EDITOR_PREFERENCES": "Editor preferences",
     "AI_CHAT_TOOL_EDITOR_DOCS": "Editor docs",

tracking-repos.json

@@ -1,5 +1,5 @@
 {
   "phoenixPro": {
-    "commitID": "2f0558b7cfccd51d8b12966f2bc7ef73a02dbbd6"
+    "commitID": "033ca7b4524c0e01a5d412893f5aee99a164a5e5"
   }
 }