From 384793bde7ebcd3040e2bb3be0149a31d2c10c2a Mon Sep 17 00:00:00 2001 From: abose Date: Sat, 5 Sep 2026 12:40:27 +0530 Subject: [PATCH 1/6] fix(ai): make the editor tools discoverable and state what the user is on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The system prompt told the model to call getEditorState first and to reach for takeScreenshot / execJsInLivePreview "listed below", but none of the phoenix-editor tools were in its tool list: they sat behind ToolSearch. It was being told to use tools it could not see, so it fell back to what it could — an Explore subagent grepping the project — and only discovered the editor tools later, usually to verify work already finished. - alwaysLoad on the four "look at what the user is looking at" tools: getEditorState, takeScreenshot, execJsInLivePreview, controlEditor. ~1.2k tokens on the cached system block; one ToolSearch round-trip costs a whole model turn, so it pays for itself immediately. - searchHint on all ten, worded in the terms someone would search with (browser, rendered page, DOM, screenshot, responsive) rather than the "live preview" jargon, so the six deferred ones stay findable. - Prepend one line naming the active file, the unsaved buffers and what the live preview renders. The panel assembles it, since the data and the context chips both live there; this side only renders the prose. Says plainly when a list is complete, because told only that a list *can* be truncated the model re-checks with getEditorState — the exact lookup this is meant to save. Measured on the same prompt against a page with a runtime-only bug: baseline +alwaysLoad +both ToolSearch calls 2 0 0 first move Explore↴ Explore↴ the right file model turns 10 9 6-9 wall time 85.5s 76.1s 17-27s cost $0.2335 $0.2960 $0.08-0.11 The line itself costs ~106 tokens typically, ~195 worst case. --- src-node/claude-code-agent.js | 45 ++++++++++++++++++++++++++++++- src-node/mcp-editor-tools.js | 50 +++++++++++++++++++++++++++++------ 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src-node/claude-code-agent.js b/src-node/claude-code-agent.js index c8bf7a6140..06e0cfb5b5 100644 --- a/src-node/claude-code-agent.js +++ b/src-node/claude-code-agent.js @@ -315,6 +315,39 @@ function _formatAnswers(answer) { return answerText.trim(); } +/** + * Render the editor context the panel sent into the line prepended to the + * prompt. The panel assembles it because that is where the data and the + * user's context chips already live; this only turns it into prose. Returns + * "" when the panel sent nothing, i.e. the user dismissed those chips. + */ +function _buildEditorContextLine(ctx) { + if (!ctx || (!ctx.activeFile && !ctx.livePreviewFile)) { + return ""; + } + const parts = ["Editor state (auto-supplied, no tool call needed):"]; + if (ctx.activeFile) { + parts.push("the user is editing " + ctx.activeFile + "."); + if (ctx.unsaved) { + parts.push("Unsaved, so stale on disk: " + ctx.unsaved + "."); + } + } + if (ctx.livePreviewFile) { + parts.push(ctx.livePreviewFile === ctx.activeFile + ? "The live preview is showing that same file." + : "The live preview is showing " + ctx.livePreviewFile + "."); + } + // Say plainly when the lists are complete. Left merely to infer it, the + // agent calls getEditorState to check — the exact lookup this line is + // here to save. + parts.push(ctx.truncated + ? "Trust this over searching for it yourself; call getEditorState for the names cut " + + "from a list, or if you need the cursor, the selection or a fresher view." + : "That is the complete set. Trust it over searching or double-checking; call " + + "getEditorState only if you need the cursor, the selection or a fresher view."); + return parts.join(" "); +} + /** * Detect whether a PostToolUse `tool_response` represents an error result. * Used to suppress diff-card painting when the SDK's native Edit/Write itself @@ -855,7 +888,8 @@ exports.checkAvailability = async function (opts) { * aiProgress, aiTextStream, aiToolEdit, aiError, aiComplete */ exports.sendPrompt = async function (params) { - const { prompt, projectPath, sessionAction, model, locale, selectionContext, images, envOverrides, permissionMode, additionalDirectories } = params; + const { prompt, projectPath, sessionAction, model, locale, selectionContext, editorContext, + images, envOverrides, permissionMode, additionalDirectories } = params; const requestId = Date.now().toString(36) + Math.random().toString(36).slice(2, 7); // Handle session @@ -874,6 +908,12 @@ exports.sendPrompt = async function (params) { currentAbortController = new AbortController(); + // Prepend what the user is looking at. The panel knows the active file, + // the unsaved buffers and the live preview target for certain, so stating + // them costs one line and removes the reason to go hunting: without it + // the model opens by grepping the project for a file already on screen. + const editorContextLine = _buildEditorContextLine(editorContext); + // Prepend selection context to the prompt if available let enrichedPrompt = prompt; if (selectionContext) { @@ -896,6 +936,9 @@ exports.sendPrompt = async function (params) { " to read the selected content if needed." + previewSnippet + "\n" + prompt; } } + if (editorContextLine) { + enrichedPrompt = editorContextLine + "\n\n" + enrichedPrompt; + } // Run the query asynchronously — don't await here so we return requestId immediately _runQuery(requestId, enrichedPrompt, projectPath, model, currentAbortController.signal, locale, images, envOverrides, permissionMode, additionalDirectories) diff --git a/src-node/mcp-editor-tools.js b/src-node/mcp-editor-tools.js index 287c865a4a..2c08cf5ae4 100644 --- a/src-node/mcp-editor-tools.js +++ b/src-node/mcp-editor-tools.js @@ -154,7 +154,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } return _maybeAppendHint(result, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "which file the user has open in Phoenix Code editor, plus cursor, selection, and what the live preview (an embedded browser rendering their HTML or Markdown) is showing" + } ); const takeScreenshotTool = sdkModule.tool( @@ -219,7 +223,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "screenshot the user's Phoenix Code editor app window, or the page rendered in their live preview browser" + } ); const execJsInLivePreviewTool = sdkModule.tool( @@ -265,7 +273,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "run JS in the user's live preview browser to inspect the rendered page's DOM, console or JS state" + } ); const controlEditorTool = sdkModule.tool( @@ -330,7 +342,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "open, close or switch files in Phoenix Code, toggle the live preview browser" + } ); const resizeLivePreviewTool = sdkModule.tool( @@ -364,7 +380,10 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + searchHint: "resize the user's live preview browser viewport to check a responsive layout" + } ); const waitTool = sdkModule.tool( @@ -383,7 +402,10 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + searchHint: "pause before re-checking the user's live preview browser" + } ); const execJsInEditorTool = sdkModule.tool( @@ -457,6 +479,9 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; } return _maybeAppendHint(toolResult, hasClarification); + }, + { + searchHint: "run JS against Phoenix Code's own editor API, not the page in its live preview" } ); @@ -527,6 +552,9 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; } return _maybeAppendHint(toolResult, hasClarification); + }, + { + searchHint: "read or change the user's Phoenix Code editor preferences" } ); @@ -574,7 +602,10 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + searchHint: "look up Phoenix Code editor feature or API documentation" + } ); const getUserClarificationTool = sdkModule.tool( @@ -607,7 +638,10 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) content: [{ type: "text", text: "No clarification queued." }] }; }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + searchHint: "read a follow-up the user typed into this conversation while you were still working" + } ); return sdkModule.createSdkMcpServer({ From 738bd4c66432a58f531efbabf4286f1feca36ae6 Mon Sep 17 00:00:00 2001 From: abose Date: Sat, 5 Sep 2026 12:44:56 +0530 Subject: [PATCH 2/6] build: update pro deps --- tracking-repos.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracking-repos.json b/tracking-repos.json index fbcab96286..05dcc264f0 100644 --- a/tracking-repos.json +++ b/tracking-repos.json @@ -1,5 +1,5 @@ { "phoenixPro": { - "commitID": "2f0558b7cfccd51d8b12966f2bc7ef73a02dbbd6" + "commitID": "3c6529a90468775773ae512157964701f402c221" } } From d4eeee716654beb6d8f45f037c73fea5d6ef2bf1 Mon Sep 17 00:00:00 2001 From: abose Date: Sat, 5 Sep 2026 14:06:08 +0530 Subject: [PATCH 3/6] feat(ai): search unsaved buffers, and load the resize tool up front MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two measured changes and one wording correction, all from the A/B runs recorded in phoenix-pro/unshipped/ai-panel-efficiency-notes.md. - searchEditorBuffers: regex search over the UNSAVED open files only. Those are the only files where the model's Grep is wrong — Grep reads disk, and disk is stale for a buffer the user has edited but not saved. Read/Edit are already buffer-safe (the agent flushes first); search was the one gap. The response names the files it covered and tells the model to Grep the rest, and when nothing is unsaved it says Grep is authoritative and does no work. Deliberately not a project scan: that would run on the UI thread and re-do what ripgrep already does off-thread. - resizeLivePreview is alwaysLoad. Responsive checks are a core use case and it cost two ToolSearch round-trips every time: same fix, 3 fewer turns, ~10s faster per responsive task. - Context line names which tools the "unsaved" note applies to. "Unsaved, so stale on disk" made the model refuse Edit and drive DocumentManager by hand through execJsInEditor — no edit card, no undo, 4x the cost. Now: "Read and Edit see the unsaved text as normal; Grep does not, so use searchEditorBuffers to search these." --- src-node/claude-code-agent.js | 9 +++++- src-node/mcp-editor-tools.js | 52 +++++++++++++++++++++++++++++++++-- src/nls/root/strings.js | 2 ++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src-node/claude-code-agent.js b/src-node/claude-code-agent.js index 06e0cfb5b5..82aea2933d 100644 --- a/src-node/claude-code-agent.js +++ b/src-node/claude-code-agent.js @@ -329,7 +329,11 @@ function _buildEditorContextLine(ctx) { if (ctx.activeFile) { parts.push("the user is editing " + ctx.activeFile + "."); if (ctx.unsaved) { - parts.push("Unsaved, so stale on disk: " + ctx.unsaved + "."); + // Read and Edit are buffer-safe here (the agent flushes the buffer + // first); only Grep sees stale disk. Saying "stale on disk" without that + // steered the model off Edit onto the editor API — no edit card, no undo. + parts.push("Unsaved (Read and Edit see the unsaved text as normal; Grep does not, so " + + "use searchEditorBuffers to search these): " + ctx.unsaved + "."); } } if (ctx.livePreviewFile) { @@ -1421,6 +1425,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale, "WebFetch", "WebSearch", "EnterPlanMode", "ExitPlanMode", "mcp__phoenix-editor__getEditorState", + "mcp__phoenix-editor__searchEditorBuffers", "mcp__phoenix-editor__takeScreenshot", "mcp__phoenix-editor__execJsInLivePreview", "mcp__phoenix-editor__execJsInEditor", @@ -1442,6 +1447,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale, " files to answer questions. Do not modify files.", tools: ["Read", "Glob", "Grep", "mcp__phoenix-editor__getEditorState", + "mcp__phoenix-editor__searchEditorBuffers", "mcp__phoenix-editor__takeScreenshot", "mcp__phoenix-editor__execJsInLivePreview", "mcp__phoenix-editor__editorDocs"] @@ -1454,6 +1460,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale, " only for new files.", tools: ["Read", "Edit", "Write", "Glob", "Grep", "mcp__phoenix-editor__getEditorState", + "mcp__phoenix-editor__searchEditorBuffers", "mcp__phoenix-editor__takeScreenshot", "mcp__phoenix-editor__execJsInLivePreview", "mcp__phoenix-editor__execJsInEditor", diff --git a/src-node/mcp-editor-tools.js b/src-node/mcp-editor-tools.js index 2c08cf5ae4..cca2a8d21f 100644 --- a/src-node/mcp-editor-tools.js +++ b/src-node/mcp-editor-tools.js @@ -54,7 +54,8 @@ const EXEC_PEER_TIMEOUT_MS = { getEditorState: 5000, takeScreenshot: 15000, controlEditor: 5000, - resizeLivePreview: 5000 + resizeLivePreview: 5000, + searchEditorBuffers: 3000 }; // Floor for caller-provided timeouts (e.g. execJsInLivePreview's @@ -161,6 +162,52 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } ); + const searchEditorBuffersTool = sdkModule.tool( + "searchEditorBuffers", + "Regex search over the UNSAVED open files only — the ones the editor-state line at the top of " + + "the prompt lists as unsaved. Those are the only files where Grep is wrong: Grep reads disk, and " + + "disk is stale for a buffer the user has edited but not saved. Use Grep for everything else; it " + + "is faster and covers the whole project. Only call this when the editor-state line names unsaved " + + "files. Returns matches {file, line, text}, searchedFiles (what this actually covered) and truncated.", + { + pattern: z.string().describe("Regex (default) or literal text to find"), + isRegex: z.boolean().optional().describe("false to match the pattern literally. Default true"), + caseSensitive: z.boolean().optional().describe("Default false"), + fileGlob: z.string().optional().describe("Limit to matching files, e.g. *.css"), + maxResults: z.number().optional().describe("Cap on matches returned. Default 50, max 200") + }, + async function (args) { + let result; + try { + const found = await _execPeerWithTimeout(nodeConnector, "searchEditorBuffers", + args || {}, "searchEditorBuffers"); + let text; + if (found && found.error) { + text = JSON.stringify(found); + } else if (!found || !found.searchedFiles || !found.searchedFiles.length) { + text = "No unsaved files, so nothing in the editor differs from disk. Use Grep — " + + "it is authoritative for the whole project right now."; + } else { + text = JSON.stringify(found) + + "\n\nThis searched ONLY the unsaved files in searchedFiles. Every other file " + + "matches disk — use Grep for the rest of the project."; + } + result = { content: [{ type: "text", text: text }] }; + } catch (err) { + result = { + content: [{ type: "text", text: "Error searching unsaved files: " + err.message }], + isError: true + }; + } + return _maybeAppendHint(result, hasClarification); + }, + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "search the unsaved editor buffers, where Grep would see stale disk content" + } + ); + const takeScreenshotTool = sdkModule.tool( "takeScreenshot", "Take a screenshot of the Phoenix Code editor application window (or a region within it). " + @@ -382,6 +429,7 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }, { annotations: { readOnlyHint: true }, + alwaysLoad: true, searchHint: "resize the user's live preview browser viewport to check a responsive layout" } ); @@ -646,7 +694,7 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) return sdkModule.createSdkMcpServer({ name: "phoenix-editor", - tools: [getEditorStateTool, takeScreenshotTool, execJsInLivePreviewTool, + tools: [getEditorStateTool, searchEditorBuffersTool, takeScreenshotTool, execJsInLivePreviewTool, execJsInEditorTool, editorPreferencesTool, editorDocsTool, controlEditorTool, resizeLivePreviewTool, waitTool, getUserClarificationTool] }); diff --git a/src/nls/root/strings.js b/src/nls/root/strings.js index cd283fb661..a45d9cbae1 100644 --- a/src/nls/root/strings.js +++ b/src/nls/root/strings.js @@ -2659,6 +2659,8 @@ define({ "AI_CHAT_TOOL_SCREENSHOT_LIVE_PREVIEW": "live preview", "AI_CHAT_TOOL_SCREENSHOT_FULL_EDITOR": "the full editor", "AI_CHAT_TOOL_LIVE_PREVIEW_JS": "Inspecting preview", + "AI_CHAT_TOOL_SEARCH_UNSAVED": "Search unsaved files", + "AI_CHAT_TOOL_SEARCH_UNSAVED_FOR": "Search unsaved files: {0}", "AI_CHAT_TOOL_EDITOR_JS": "Inspecting editor", "AI_CHAT_TOOL_EDITOR_PREFERENCES": "Editor preferences", "AI_CHAT_TOOL_EDITOR_DOCS": "Editor docs", From 69b3152bfc3ec25ee67ffa5e29a7159bad96c14d Mon Sep 17 00:00:00 2001 From: abose Date: Sat, 5 Sep 2026 15:56:40 +0530 Subject: [PATCH 4/6] feat(mcp): run, track, file and compare the AI panel model tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four phoenix-builder tools so a running Claude session can be asked "run the AI test suite" and do the rest. The session is the runner and the judge; the tools do the deterministic parts. - run_ai_test_suite: installs the fixture, records git revs / CLI version / connected instance, opens a run record, and returns the runner briefing plus the documents for exactly the tests in scope — quick (default), all, a suite name, explicit test ids, or resumeRunId for a stopped run. - ai_test_progress: called after every test; answers "how far along is it", lists runs, and stops a run on request. Progress lives in reports/runs/. - save_ai_test_report: writes reports/latest.md, overwriting the previous run — git history is the archive, baseline.md is never touched. A stopped run is saved with a Partial section; a completed run's progress record is removed. - compare_ai_test_reports: diffs latest.md against baseline.md by default, or against the last committed latest.md ("previous"), flagging REGRESSION / quality drop / slower per the suite's thresholds. Reads the merged "PASS · poor" Result column and the older two-column form. The suite itself lives in phoenix-pro under unit-tests/ai_model_tests/. CLAUDE.md tells any session what to do when asked; the MCP README lists the tools. .eslintignore excludes the suite's fixture pages — they are deliberately broken and are test data, not code. --- .eslintignore | 3 + CLAUDE.md | 3 + phoenix-builder-mcp/README.md | 12 + phoenix-builder-mcp/mcp-tools.js | 453 +++++++++++++++++++++++++++++++ 4 files changed, 471 insertions(+) diff --git a/.eslintignore b/.eslintignore index c5dee2a504..506208f040 100644 --- a/.eslintignore +++ b/.eslintignore @@ -25,3 +25,6 @@ test/**/node_modules/**/*.js test/virtual-server-test.js test/spec/ESLintExtensionTest-files + +# AI panel model-test fixtures: deliberately broken pages the tests operate on, not code +src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/fixtures/** diff --git a/CLAUDE.md b/CLAUDE.md index 0f095c4618..9ffecaba2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,6 +48,9 @@ Use `exec_js` to run JS in the Phoenix browser runtime. jQuery `$()` is global. **Check logs:** `get_browser_console_logs` with `filter` regex (e.g. `"AI UI"`, `"error"`) and `tail` — includes both browser console and Node.js (PhNode) logs. Use `get_terminal_logs` for Electron process output (only available if Phoenix was launched via `start_phoenix`). +## AI model tests (behavioural tests of the AI panel) +When asked to "run the AI test suite" / "run the model tests" / "run EC-1 and UB-2": call `run_ai_test_suite` (phoenix-builder MCP) with `suite` (`quick` | `all` | a suite name), or `tests` for specific IDs, or `resumeRunId` to continue. It installs the fixture, opens a run record, and returns the briefing plus the test documents from `src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/`. You are the runner and the judge — follow them exactly, deterministic checks first. After **every** test call `ai_test_progress` and tell the user one progress line. If the user says stop: `ai_test_progress({ runId, stop: true })`, then save. Finish with `save_ai_test_report`, then `compare_ai_test_reports({})`, and tell the user the report path, PASS/FAIL counts, any regressions, and the Observations section. + ## Writing Tests - **Never use `awaits(number)`** (fixed-time waits) in tests — they cause flaky failures. Always use `awaitsFor(condition)` to wait for a specific condition to become true. - Use `editor.*` APIs (e.g. `editor.document.getText()`, `editor.getCursorPos()`, `editor.setSelection()`) instead of accessing `editor._codeMirror` directly. diff --git a/phoenix-builder-mcp/README.md b/phoenix-builder-mcp/README.md index 9d088b8535..6e27c64787 100644 --- a/phoenix-builder-mcp/README.md +++ b/phoenix-builder-mcp/README.md @@ -97,6 +97,18 @@ Reloads the Phoenix app. Prompts to save unsaved files before reloading. ### `force_reload_phoenix` Force-reloads the Phoenix app without saving unsaved changes. +### `run_ai_test_suite` +Starts (or resumes) the AI panel model tests and hands the session everything it needs: installs the fixture project, gathers git revisions / CLI version / connected instance, opens a run record, and returns the runner briefing plus the test documents from `src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/`. The session is the runner and the judge. `suite`: `quick` (default), `all`, or a suite name; or `tests: ["EC-1","UB-2"]` for specific tests; or `resumeRunId` to continue a stopped run. Ask Claude: *"run the AI test suite"*, *"run just the plan-mode tests"*, *"run EC-1 and UB-2"*. + +### `ai_test_progress` +Called by the runner after every test to record the result; also answers *"how far along is it?"* (`{ runId }`), lists runs (`{}`), and stops a run (`{ runId, stop: true }`). Progress lives in `reports/runs/.json`, which you can open at any time. + +### `save_ai_test_report` +Writes the finished report to `reports/latest.md` inside the suite folder, overwriting the previous run (git history keeps earlier runs; `baseline.md` is never touched). A stopped run is saved with a Partial section listing the unrun tests. + +### `compare_ai_test_reports` +Diffs two reports test by test — by default `latest.md` against `baseline.md`, or `against: "previous"` for the last committed run — and flags regressions, quality drops, and slower runs using the thresholds in `model_tests.md`. + ## Typical Claude Code workflow ``` diff --git a/phoenix-builder-mcp/mcp-tools.js b/phoenix-builder-mcp/mcp-tools.js index 070ea24a91..6a9ff65ec7 100644 --- a/phoenix-builder-mcp/mcp-tools.js +++ b/phoenix-builder-mcp/mcp-tools.js @@ -1,4 +1,9 @@ import { z } from "zod"; +import fs from "fs"; +import os from "os"; +import path from "path"; +import { execSync } from "child_process"; +import { fileURLToPath } from "url"; const DEFAULT_MAX_CHARS = 10000; @@ -15,6 +20,145 @@ function _trimToCharBudget(lines, maxChars) { return { lines: lines.slice(startIdx), trimmed: startIdx }; } +// ---- AI model test suite --------------------------------------------------- +// The suites are markdown procedures run by a Claude session against the +// connected Phoenix instance; the session is the runner and the judge. These +// tools do the deterministic parts — install the fixture, gather the +// environment, hand over the documents, file the report — so that "run the AI +// test suite" is a single request. + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const AI_TESTS_DIR = path.join(REPO_ROOT, "src", "extensionsIntegrated", "phoenix-pro", + "unit-tests", "ai_model_tests"); +const AI_TESTS_FIXTURE_DEST = path.join(os.homedir(), "Documents", + "Phoenix Code Experimental Build", "ai-model-tests", "taskboard"); +const AI_TEST_SUITES = { + "editor-context": "suite-editor-context.md", + "tool-discovery": "suite-tool-discovery.md", + "unsaved-buffers": "suite-unsaved-buffers.md", + "self-sufficiency": "suite-self-sufficiency.md", + "bug-fixing": "suite-bug-fixing.md", + "plan-mode": "suite-plan-mode.md", + "permissions": "suite-permissions.md" +}; +// The four model runs that have caught every regression seen so far, plus the +// free deterministic/piggyback checks. See model_tests.md, "Deterministic first". +const AI_TEST_QUICK = { + suites: ["editor-context", "unsaved-buffers", "self-sufficiency", "bug-fixing"], + tests: ["UB-1", "EC-5", "EC-2", "SS-4", "EC-1", "UB-2", "SS-1", "BF-1"] +}; + +function _gitInfo(cwd) { + try { + const rev = execSync("git rev-parse --short HEAD", { cwd, stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); + const branch = execSync("git branch --show-current", { cwd, stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); + const dirty = execSync("git status --porcelain", { cwd, stdio: ["ignore", "pipe", "ignore"] }).toString().trim() ? " (uncommitted changes)" : ""; + return `${rev} on ${branch}${dirty}`; + } catch (e) { + return "unknown"; + } +} + +function _claudeCliVersion() { + try { + return execSync("claude --version", { stdio: ["ignore", "pipe", "ignore"], timeout: 5000 }).toString().trim(); + } catch (e) { + return "unknown (read it from the panel's logs)"; + } +} + +// Two committed files only: baseline.md (the reference, edited by hand) and +// latest.md (overwritten every run — git history is the archive). Anything +// else in the folder is listed but not special. +const AI_TEST_REPORT_LATEST = "latest.md"; +const AI_TEST_REPORT_BASELINE = "baseline.md"; +function _listReports() { + const dir = path.join(AI_TESTS_DIR, "reports"); + if (!fs.existsSync(dir)) { return []; } + return fs.readdirSync(dir).filter(f => f.endsWith(".md")).sort(); +} +// The committed version of a report file, for `against: "previous"`. +function _gitHeadVersion(fileName) { + const proDir = path.join(REPO_ROOT, "src", "extensionsIntegrated", "phoenix-pro"); + const rel = path.posix.join("unit-tests", "ai_model_tests", "reports", fileName); + try { + return execSync(`git show HEAD:${rel}`, { cwd: proDir, stdio: ["ignore", "pipe", "ignore"] }).toString(); + } catch (e) { + return null; + } +} + +function _countTests(markdown) { + return (markdown.match(/^## [A-Z]{2}-\d+/gm) || []).length; +} + +function _todayStamp() { + return new Date().toISOString().slice(0, 10); +} + +const AI_TEST_RUNS_DIR = path.join(AI_TESTS_DIR, "reports", "runs"); + +function _runPath(runId) { + return path.join(AI_TEST_RUNS_DIR, runId.replace(/[^a-zA-Z0-9-]+/g, "-") + ".json"); +} +function _saveRun(run) { + fs.mkdirSync(AI_TEST_RUNS_DIR, { recursive: true }); + fs.writeFileSync(_runPath(run.runId), JSON.stringify(run, null, 2) + "\n", "utf8"); +} +function _loadRun(runId) { + const p = _runPath(runId); + return fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : null; +} +function _listRuns() { + if (!fs.existsSync(AI_TEST_RUNS_DIR)) { return []; } + return fs.readdirSync(AI_TEST_RUNS_DIR).filter(f => f.endsWith(".json")).map(f => f.slice(0, -5)).sort().reverse(); +} +function _countResults(results) { + const c = { PASS: 0, FAIL: 0, FLAKY: 0, BLOCKED: 0, good: 0, acceptable: 0, poor: 0 }; + for (const r of results) { + if (c[r.invariants] !== undefined) { c[r.invariants]++; } + if (c[r.judgement] !== undefined) { c[r.judgement]++; } + } + return c; +} +// Every "## XX-n — title" heading across the suite files, keyed by id. +function _loadTestCatalog() { + const catalog = {}; + for (const [suite, file] of Object.entries(AI_TEST_SUITES)) { + const full = path.join(AI_TESTS_DIR, file); + if (!fs.existsSync(full)) { continue; } + for (const m of fs.readFileSync(full, "utf8").matchAll(/^## ([A-Z]{2}-\d+)\s*—\s*(.+)$/gm)) { + catalog[m[1]] = { suite, file, title: m[2].trim() }; + } + } + return catalog; +} +// Rows of a report's "## Results" table, keyed by test id. Tolerates the +// baseline's range rows ("EC-2..EC-6") by keying them as written. +function _parseResults(markdown) { + const out = {}; + const sec = markdown.split(/^## Results/m)[1]; + if (!sec) { return out; } + for (const line of sec.split("\n")) { + if (!line.startsWith("|") || /^\|\s*-/.test(line) || /^\|\s*Suite/.test(line)) { continue; } + const cells = line.split("|").slice(1, -1).map(c => c.trim()); + if (cells.length < 4) { continue; } + const id = cells[1]; + if (!/^[A-Z]{2}-\d/.test(id)) { continue; } + const num = v => { const m = String(v || "").match(/[\d.]+/); return m ? parseFloat(m[0]) : null; }; + // Result cell is "PASS · poor" (or older reports: separate Invariants + // and Judgement columns). Detect which by whether column 3 looks like a + // judgement word. + const parts = (cells[2] || "").split(/\s*[·/]\s*/); + const separate = /^(good|acceptable|poor|n\/a|—|-)$/i.test(cells[3] || ""); + const inv = parts[0].split(/\s/)[0].toUpperCase(); + const jud = (separate ? cells[3] : (parts[1] || "")).split(/\s/)[0].toLowerCase(); + const off = separate ? 4 : 3; + out[id] = { inv, jud, tools: num(cells[off]), turns: num(cells[off + 1]), timeS: num(cells[off + 2]), cost: num(cells[off + 3]) }; + } + return out; +} + export function registerTools(server, processManager, wsControlServer, phoenixDesktopPath) { server.tool( "start_phoenix", @@ -512,4 +656,313 @@ export function registerTools(server, processManager, wsControlServer, phoenixDe }; } ); + + server.tool( + "run_ai_test_suite", + "Start (or resume) the AI panel model tests. You (the calling session) are the runner and the judge: " + + "this installs the fixture, gathers the environment, opens a run record for progress tracking, and " + + "returns the runner briefing plus the test documents to follow step by step. Scope: suite = 'quick' " + + "(default, ~4 model runs + free checks) | 'all' (~20 model runs) | one of " + + Object.keys(AI_TEST_SUITES).join(", ") + "; or tests = explicit IDs like ['EC-1','UB-2'] to run only " + + "those. resumeRunId continues an earlier run's remaining tests. After every test call ai_test_progress; " + + "when done (or told to stop) call save_ai_test_report and tell the user where it is.", + { + suite: z.string().optional().describe("quick | all | " + Object.keys(AI_TEST_SUITES).join(" | ")), + tests: z.array(z.string()).optional().describe("Explicit test IDs to run, e.g. [\"EC-1\",\"UB-2\"]. Overrides suite."), + resumeRunId: z.string().optional().describe("Run id from a previous briefing; runs only its remaining tests") + }, + async ({ suite, tests, resumeRunId }) => { + if (!fs.existsSync(path.join(AI_TESTS_DIR, "model_tests.md"))) { + return { content: [{ type: "text", text: "Test suite not found at " + AI_TESTS_DIR + + ". Is phoenix-pro checked out inside this phoenix repo?" }], isError: true }; + } + const catalog = _loadTestCatalog(); // { "EC-1": { suite, file, title }, ... } + + let which, planned, run; + if (resumeRunId) { + run = _loadRun(resumeRunId); + if (!run) { + return { content: [{ type: "text", text: "No run record '" + resumeRunId + "'. Known runs: " + + _listRuns().join(", ") }], isError: true }; + } + const done = new Set(run.results.map(r => r.test)); + planned = run.planned.filter(t => !done.has(t)); + which = run.scope; + run.status = "running"; + run.resumedAt = new Date().toISOString(); + } else if (tests && tests.length) { + const unknown = tests.filter(t => !catalog[t.toUpperCase()]); + if (unknown.length) { + return { content: [{ type: "text", text: "Unknown test id(s): " + unknown.join(", ") + + ". Known: " + Object.keys(catalog).join(", ") }], isError: true }; + } + planned = tests.map(t => t.toUpperCase()); + which = "selected"; + } else { + which = (suite || "quick").trim().toLowerCase(); + if (which !== "quick" && which !== "all" && !AI_TEST_SUITES[which]) { + return { content: [{ type: "text", text: "Unknown suite '" + which + "'. Use quick, all, or one of: " + + Object.keys(AI_TEST_SUITES).join(", ") }], isError: true }; + } + planned = which === "quick" ? AI_TEST_QUICK.tests.slice() + : Object.keys(catalog).filter(id => which === "all" || catalog[id].suite === which); + } + if (!run) { + run = { + runId: _todayStamp() + "-" + new Date().toTimeString().slice(0, 5).replace(":", "") + "-" + which, + scope: which, planned, results: [], status: "running", + startedAt: new Date().toISOString(), env: {} + }; + } + + // H0: install the fixture. Deterministic, so do it here rather than ask the runner. + const fixtureSrc = path.join(AI_TESTS_DIR, "fixtures", "taskboard"); + fs.mkdirSync(AI_TESTS_FIXTURE_DEST, { recursive: true }); + for (const f of fs.readdirSync(fixtureSrc)) { + fs.copyFileSync(path.join(fixtureSrc, f), path.join(AI_TESTS_FIXTURE_DEST, f)); + } + + const instances = wsControlServer.getConnectedInstances(); + const electron = instances.filter(i => i.startsWith("phoenix-electron-")); + const proDir = path.join(REPO_ROOT, "src", "extensionsIntegrated", "phoenix-pro"); + run.env = { phoenix: _gitInfo(REPO_ROOT), phoenixPro: _gitInfo(proDir), claudeCli: _claudeCliVersion(), instance: electron[0] || null }; + _saveRun(run); + + const suiteFiles = [...new Set(planned.map(id => catalog[id].file))]; + const docs = suiteFiles.map(file => ({ file, md: fs.readFileSync(path.join(AI_TESTS_DIR, file), "utf8") })); + const index = fs.readFileSync(path.join(AI_TESTS_DIR, "model_tests.md"), "utf8"); + const reports = _listReports(); + const baseline = reports.includes(AI_TEST_REPORT_BASELINE) ? AI_TEST_REPORT_BASELINE : null; + const reportName = AI_TEST_REPORT_LATEST; + + const briefing = [ + "# AI model tests — runner briefing", + "", + "You are the runner and the judge. Follow the documents below exactly. Do every step as written,", + "check every invariant, apply every judgement rubric, and write one or two sentences of reasoning", + "per test. Run the deterministic checks first, and stop if one fails.", + "", + "## Progress, stopping, and the user", + `- Run id: **${run.runId}**. After EVERY test, call ai_test_progress({ runId, test, invariants, judgement,`, + " tools, turns, timeMs, cost, reasoning }). It returns done/total and what remains.", + "- After every test, tell the user one line: ` — n/total done`.", + `- The run record the user can open at any time: ${_runPath(run.runId)}`, + "- If the user says stop: finish the step you are in, call ai_test_progress({ runId, stop: true }),", + " then save_ai_test_report with what you have — the report is marked partial and lists what was not run.", + " It can be resumed later with run_ai_test_suite({ resumeRunId }).", + "", + "## What will run", + `- Scope: **${which}** — ${planned.length} test(s)${resumeRunId ? " remaining" : ""}: ${planned.join(", ")}`, + ...suiteFiles.map(f => ` - ${f}`), + which === "quick" ? "- Quick means ONLY the listed tests. Skip everything else in those documents." : "", + which === "selected" ? "- Selected means ONLY the listed tests. Skip everything else in those documents." : "", + "", + "## Environment (put this at the top of the report)", + "- Runner model: ", + "- Model under test: ", + `- phoenix: ${run.env.phoenix}`, + `- phoenix-pro: ${run.env.phoenixPro}`, + `- Claude CLI: ${run.env.claudeCli}`, + `- Connected instances: ${instances.length ? instances.join(", ") : "none — start Phoenix first"}`, + electron.length ? `- Use instance: ${electron[0]}` : "- No phoenix-electron-* instance is connected. Ask the user to open the desktop app, then call get_phoenix_status.", + "", + "## Where things are", + `- Fixture installed (H0 done for you): ${AI_TESTS_FIXTURE_DEST}`, + `- In the documents, replace with: ${AI_TESTS_FIXTURE_DEST}`, + `- Transcript folder for H10: ~/.claude/projects/${AI_TESTS_FIXTURE_DEST.replace(/[\/ ]/g, "-")}`, + `- Test documents: ${AI_TESTS_DIR}`, + `- Reports folder: ${path.join(AI_TESTS_DIR, "reports")}`, + `- Compare against: ${baseline ? path.join(AI_TESTS_DIR, "reports", baseline) : "no baseline found — this run becomes the baseline"}`, + `- Reports in the folder: ${reports.length ? reports.join(", ") : "none"} (earlier runs live in git history of ${reportName})`, + `- Your report will be saved as: ${reportName} (overwrites the previous run) — call save_ai_test_report({ content, runId: "${run.runId}" }).`, + " Then call compare_ai_test_reports({}) to diff it against the baseline — or { against: \"previous\" } for the last committed run — and relay the result.", + "", + "---", + "", + "# Document 1 of " + (docs.length + 1) + ": model_tests.md (rules, harness, report format)", + "", + index, + ...docs.flatMap((d, i) => ["", "---", "", `# Document ${i + 2} of ${docs.length + 1}: ${d.file}`, "", d.md]) + ].filter(l => l !== null).join("\n"); + + return { content: [{ type: "text", text: briefing }] }; + } + ); + + server.tool( + "ai_test_progress", + "Record one test's result during an AI model test run, or read a run's progress, or stop it. With test + " + + "result fields: appends the result and returns done/total/remaining. With only runId: returns current " + + "progress (use this to answer 'how far along is it?'). With stop: true: marks the run stopped and returns " + + "what was not run. With no runId at all: lists known runs.", + { + runId: z.string().optional().describe("Run id from the briefing"), + test: z.string().optional().describe("Test id, e.g. EC-1"), + invariants: z.enum(["PASS", "FAIL", "FLAKY", "BLOCKED"]).optional(), + judgement: z.enum(["good", "acceptable", "poor", "n/a"]).optional(), + tools: z.number().optional(), turns: z.number().optional(), + timeMs: z.number().optional(), cost: z.number().optional(), + reasoning: z.string().optional().describe("One or two sentences: what the AI did and why that verdict"), + stop: z.boolean().optional().describe("Mark the run stopped (user asked to stop)") + }, + async (args) => { + if (!args.runId) { + const runs = _listRuns().map(id => { const r = _loadRun(id); return `${id}: ${r.status}, ${r.results.length}/${r.planned.length}`; }); + return { content: [{ type: "text", text: runs.length ? "Known runs:\n" + runs.join("\n") : "No runs recorded." }] }; + } + const run = _loadRun(args.runId); + if (!run) { + return { content: [{ type: "text", text: "No run record '" + args.runId + "'. Known: " + _listRuns().join(", ") }], isError: true }; + } + if (args.test) { + const id = args.test.toUpperCase(); + run.results = run.results.filter(r => r.test !== id); // a rerun replaces + run.results.push({ + test: id, invariants: args.invariants || "BLOCKED", judgement: args.judgement || "n/a", + tools: args.tools, turns: args.turns, timeMs: args.timeMs, cost: args.cost, + reasoning: args.reasoning || "", at: new Date().toISOString() + }); + } + if (args.stop) { + run.status = "stopped"; + run.stoppedAt = new Date().toISOString(); + } else if (run.results.length >= run.planned.length && run.planned.every(t => run.results.some(r => r.test === t))) { + run.status = "complete"; + run.completedAt = new Date().toISOString(); + } + _saveRun(run); + const done = run.planned.filter(t => run.results.some(r => r.test === t)); + const remaining = run.planned.filter(t => !done.includes(t)); + const counts = _countResults(run.results); + return { content: [{ type: "text", text: JSON.stringify({ + runId: run.runId, status: run.status, done: done.length, total: run.planned.length, + remaining, counts, record: _runPath(run.runId), + progressLine: `${done.length}/${run.planned.length} done · PASS ${counts.PASS} FAIL ${counts.FAIL} FLAKY ${counts.FLAKY} BLOCKED ${counts.BLOCKED}` + + (remaining.length ? ` · next: ${remaining[0]}` : " · all done") + }, null, 2) }] }; + } + ); + + server.tool( + "save_ai_test_report", + "File an AI model test report as reports/latest.md, overwriting the previous run (git history keeps " + + "earlier runs; baseline.md is never touched). If runId is given and the run was stopped, the report is " + + "marked partial and the unrun tests are appended. Returns the saved path and the baseline to compare " + + "against — relay the path to the user.", + { + content: z.string().describe("The full report markdown, following the skeleton in model_tests.md"), + runId: z.string().optional().describe("Run id from the briefing; links the report to its progress record"), + label: z.string().optional().describe("Ignored for the filename (always latest.md); kept for compatibility") + }, + async ({ content, runId }) => { + const dir = path.join(AI_TESTS_DIR, "reports"); + fs.mkdirSync(dir, { recursive: true }); + let body = content.endsWith("\n") ? content : content + "\n"; + const run = runId ? _loadRun(runId) : null; + let partial = false; + if (run) { + const notRun = run.planned.filter(t => !run.results.some(r => r.test === t)); + if (run.status === "stopped" || notRun.length) { + partial = true; + body += `\n## Partial run\nStopped ${run.stoppedAt || "before completion"}. Not run: ${notRun.join(", ") || "none"}.\n` + + `Resume with run_ai_test_suite({ resumeRunId: "${run.runId}" }).\n`; + } + } + const full = path.join(dir, AI_TEST_REPORT_LATEST); + const hadPrevious = fs.existsSync(full); + fs.writeFileSync(full, body, "utf8"); + if (run) { + // A finished run's progress record has done its job; only a + // stopped (resumable) run keeps its file. Keeps runs/ from piling up. + if (partial) { run.report = full; _saveRun(run); } + else if (fs.existsSync(_runPath(run.runId))) { fs.unlinkSync(_runPath(run.runId)); } + } + const hasBaseline = fs.existsSync(path.join(dir, AI_TEST_REPORT_BASELINE)); + return { content: [{ type: "text", text: JSON.stringify({ + saved: full, + partial, + overwrotePrevious: hadPrevious, + compareAgainst: hasBaseline ? path.join(dir, AI_TEST_REPORT_BASELINE) : null, + tellTheUser: "Report saved to " + full + (partial ? " (partial run)" : "") + + (hadPrevious ? "; the previous run is in git history" : "") + + (hasBaseline ? "; compare with compare_ai_test_reports({}) or { against: \"previous\" }" : "") + "." + }, null, 2) }] }; + } + ); + + server.tool( + "compare_ai_test_reports", + "Diff two AI model test reports test by test: invariant changes (REGRESSION / fixed), judgement changes " + + "(quality drop / improved), and tool/turn/time deltas with the thresholds from model_tests.md (tools " + + ">1.5x or time >2x = 'slower'). Defaults: report = latest.md, against = baseline.md. against: 'previous' " + + "compares latest.md with its last committed version (git HEAD). File names or absolute paths also accepted.", + { + report: z.string().optional().describe("Report to evaluate. Default: latest.md"), + against: z.string().optional().describe("Reference: 'baseline' (default), 'previous' (git HEAD of latest.md), a file name, or a path") + }, + async ({ report, against }) => { + const dir = path.join(AI_TESTS_DIR, "reports"); + const reports = _listReports(); + const resolve = (n, fallback) => { + const pick = n || fallback; + if (!pick) { return null; } + const full = path.isAbsolute(pick) ? pick : path.join(dir, pick); + return fs.existsSync(full) ? full : null; + }; + const a = resolve(report, AI_TEST_REPORT_LATEST); + let b, bLabel; + if ((against || "").toLowerCase() === "previous") { + const prev = _gitHeadVersion(AI_TEST_REPORT_LATEST); + if (!prev) { + return { content: [{ type: "text", text: "No committed version of " + AI_TEST_REPORT_LATEST + + " in git HEAD to compare against." }], isError: true }; + } + b = path.join(os.tmpdir(), "ai-model-tests-previous-latest.md"); fs.writeFileSync(b, prev, "utf8"); + bLabel = AI_TEST_REPORT_LATEST + " @ git HEAD"; + } else { + b = resolve((against || "").toLowerCase() === "baseline" ? null : against, AI_TEST_REPORT_BASELINE); + bLabel = b ? path.basename(b) : null; + } + if (!a || !b) { + return { content: [{ type: "text", text: "Need two reports. Have: " + (reports.join(", ") || "none") + + (a ? "" : " — report not found") + (b ? "" : " — reference not found") }], isError: true }; + } + const A = _parseResults(fs.readFileSync(a, "utf8")), B = _parseResults(fs.readFileSync(b, "utf8")); + const ids = [...new Set([...Object.keys(B), ...Object.keys(A)])].sort(); + const rows = [], flags = { regression: [], fixed: [], qualityDrop: [], improved: [], slower: [], faster: [], onlyInReport: [], onlyInReference: [] }; + const rank = { good: 2, acceptable: 1, poor: 0 }; + for (const id of ids) { + const x = A[id], y = B[id]; + if (!x) { flags.onlyInReference.push(id); rows.push(`| ${id} | — | ${y.inv}/${y.jud} | not in report |`); continue; } + if (!y) { flags.onlyInReport.push(id); rows.push(`| ${id} | ${x.inv}/${x.jud} | — | new |`); continue; } + const notes = []; + if (y.inv === "PASS" && x.inv === "FAIL") { flags.regression.push(id); notes.push("**REGRESSION**"); } + if (y.inv === "FAIL" && x.inv === "PASS") { flags.fixed.push(id); notes.push("fixed"); } + if (rank[x.jud] !== undefined && rank[y.jud] !== undefined) { + if (rank[x.jud] < rank[y.jud]) { flags.qualityDrop.push(id); notes.push("quality drop"); } + if (rank[x.jud] > rank[y.jud]) { flags.improved.push(id); notes.push("improved"); } + } + if (x.tools && y.tools && x.tools > 1.5 * y.tools) { flags.slower.push(id); notes.push(`tools ${y.tools}→${x.tools}`); } + if (x.timeS && y.timeS && x.timeS > 2 * y.timeS) { if (!flags.slower.includes(id)) { flags.slower.push(id); } notes.push(`time ${y.timeS}s→${x.timeS}s`); } + if (x.timeS && y.timeS && x.timeS < 0.5 * y.timeS && x.inv === "PASS") { flags.faster.push(id); notes.push(`faster ${y.timeS}s→${x.timeS}s`); } + rows.push(`| ${id} | ${x.inv}/${x.jud} | ${y.inv}/${y.jud} | ${notes.join(", ") || "same"} |`); + } + const out = [ + `# Comparison: ${path.basename(a)} vs ${bLabel}`, + "", + `Regressions: ${flags.regression.length ? flags.regression.join(", ") : "none"}`, + `Quality drops: ${flags.qualityDrop.length ? flags.qualityDrop.join(", ") : "none"}`, + `Slower (tools >1.5x or time >2x): ${flags.slower.length ? flags.slower.join(", ") : "none"}`, + `Fixed: ${flags.fixed.join(", ") || "none"} · Improved judgement: ${flags.improved.join(", ") || "none"} · Faster: ${flags.faster.join(", ") || "none"}`, + flags.onlyInReport.length ? `Only in report (no reference): ${flags.onlyInReport.join(", ")}` : "", + flags.onlyInReference.length ? `Not run this time: ${flags.onlyInReference.join(", ")}` : "", + "", + "| Test | report (inv/judg) | reference (inv/judg) | change |", + "| --- | --- | --- | --- |", + ...rows + ].filter(Boolean).join("\n"); + return { content: [{ type: "text", text: out }] }; + } + ); + } From 2506ccc75b68217d1000ab1977abdc2c2f239ca4 Mon Sep 17 00:00:00 2001 From: abose Date: Sat, 5 Sep 2026 15:58:54 +0530 Subject: [PATCH 5/6] build: update pro deps --- tracking-repos.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracking-repos.json b/tracking-repos.json index 05dcc264f0..0a495a01f8 100644 --- a/tracking-repos.json +++ b/tracking-repos.json @@ -1,5 +1,5 @@ { "phoenixPro": { - "commitID": "3c6529a90468775773ae512157964701f402c221" + "commitID": "5700c7cd4e327894f91c7351a5c34995175e4694" } } From 22792888eea7ad2b69800289a63642ef4c632fa9 Mon Sep 17 00:00:00 2001 From: abose Date: Sat, 5 Sep 2026 16:53:01 +0530 Subject: [PATCH 6/6] build: update pro deps --- tracking-repos.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracking-repos.json b/tracking-repos.json index 0a495a01f8..283e04c7bf 100644 --- a/tracking-repos.json +++ b/tracking-repos.json @@ -1,5 +1,5 @@ { "phoenixPro": { - "commitID": "5700c7cd4e327894f91c7351a5c34995175e4694" + "commitID": "033ca7b4524c0e01a5d412893f5aee99a164a5e5" } }