diff --git a/.eslintignore b/.eslintignore index c5dee2a504..506208f040 100644 --- a/.eslintignore +++ b/.eslintignore @@ -25,3 +25,6 @@ test/**/node_modules/**/*.js test/virtual-server-test.js test/spec/ESLintExtensionTest-files + +# AI panel model-test fixtures: deliberately broken pages the tests operate on, not code +src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/fixtures/** diff --git a/CLAUDE.md b/CLAUDE.md index 0f095c4618..9ffecaba2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,6 +48,9 @@ Use `exec_js` to run JS in the Phoenix browser runtime. jQuery `$()` is global. **Check logs:** `get_browser_console_logs` with `filter` regex (e.g. `"AI UI"`, `"error"`) and `tail` — includes both browser console and Node.js (PhNode) logs. Use `get_terminal_logs` for Electron process output (only available if Phoenix was launched via `start_phoenix`). +## AI model tests (behavioural tests of the AI panel) +When asked to "run the AI test suite" / "run the model tests" / "run EC-1 and UB-2": call `run_ai_test_suite` (phoenix-builder MCP) with `suite` (`quick` | `all` | a suite name), or `tests` for specific IDs, or `resumeRunId` to continue. It installs the fixture, opens a run record, and returns the briefing plus the test documents from `src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/`. You are the runner and the judge — follow them exactly, deterministic checks first. After **every** test call `ai_test_progress` and tell the user one progress line. If the user says stop: `ai_test_progress({ runId, stop: true })`, then save. Finish with `save_ai_test_report`, then `compare_ai_test_reports({})`, and tell the user the report path, PASS/FAIL counts, any regressions, and the Observations section. + ## Writing Tests - **Never use `awaits(number)`** (fixed-time waits) in tests — they cause flaky failures. Always use `awaitsFor(condition)` to wait for a specific condition to become true. - Use `editor.*` APIs (e.g. `editor.document.getText()`, `editor.getCursorPos()`, `editor.setSelection()`) instead of accessing `editor._codeMirror` directly. diff --git a/phoenix-builder-mcp/README.md b/phoenix-builder-mcp/README.md index 9d088b8535..6e27c64787 100644 --- a/phoenix-builder-mcp/README.md +++ b/phoenix-builder-mcp/README.md @@ -97,6 +97,18 @@ Reloads the Phoenix app. Prompts to save unsaved files before reloading. ### `force_reload_phoenix` Force-reloads the Phoenix app without saving unsaved changes. +### `run_ai_test_suite` +Starts (or resumes) the AI panel model tests and hands the session everything it needs: installs the fixture project, gathers git revisions / CLI version / connected instance, opens a run record, and returns the runner briefing plus the test documents from `src/extensionsIntegrated/phoenix-pro/unit-tests/ai_model_tests/`. The session is the runner and the judge. `suite`: `quick` (default), `all`, or a suite name; or `tests: ["EC-1","UB-2"]` for specific tests; or `resumeRunId` to continue a stopped run. Ask Claude: *"run the AI test suite"*, *"run just the plan-mode tests"*, *"run EC-1 and UB-2"*. + +### `ai_test_progress` +Called by the runner after every test to record the result; also answers *"how far along is it?"* (`{ runId }`), lists runs (`{}`), and stops a run (`{ runId, stop: true }`). Progress lives in `reports/runs/.json`, which you can open at any time. + +### `save_ai_test_report` +Writes the finished report to `reports/latest.md` inside the suite folder, overwriting the previous run (git history keeps earlier runs; `baseline.md` is never touched). A stopped run is saved with a Partial section listing the unrun tests. + +### `compare_ai_test_reports` +Diffs two reports test by test — by default `latest.md` against `baseline.md`, or `against: "previous"` for the last committed run — and flags regressions, quality drops, and slower runs using the thresholds in `model_tests.md`. + ## Typical Claude Code workflow ``` diff --git a/phoenix-builder-mcp/mcp-tools.js b/phoenix-builder-mcp/mcp-tools.js index 070ea24a91..6a9ff65ec7 100644 --- a/phoenix-builder-mcp/mcp-tools.js +++ b/phoenix-builder-mcp/mcp-tools.js @@ -1,4 +1,9 @@ import { z } from "zod"; +import fs from "fs"; +import os from "os"; +import path from "path"; +import { execSync } from "child_process"; +import { fileURLToPath } from "url"; const DEFAULT_MAX_CHARS = 10000; @@ -15,6 +20,145 @@ function _trimToCharBudget(lines, maxChars) { return { lines: lines.slice(startIdx), trimmed: startIdx }; } +// ---- AI model test suite --------------------------------------------------- +// The suites are markdown procedures run by a Claude session against the +// connected Phoenix instance; the session is the runner and the judge. These +// tools do the deterministic parts — install the fixture, gather the +// environment, hand over the documents, file the report — so that "run the AI +// test suite" is a single request. + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const AI_TESTS_DIR = path.join(REPO_ROOT, "src", "extensionsIntegrated", "phoenix-pro", + "unit-tests", "ai_model_tests"); +const AI_TESTS_FIXTURE_DEST = path.join(os.homedir(), "Documents", + "Phoenix Code Experimental Build", "ai-model-tests", "taskboard"); +const AI_TEST_SUITES = { + "editor-context": "suite-editor-context.md", + "tool-discovery": "suite-tool-discovery.md", + "unsaved-buffers": "suite-unsaved-buffers.md", + "self-sufficiency": "suite-self-sufficiency.md", + "bug-fixing": "suite-bug-fixing.md", + "plan-mode": "suite-plan-mode.md", + "permissions": "suite-permissions.md" +}; +// The four model runs that have caught every regression seen so far, plus the +// free deterministic/piggyback checks. See model_tests.md, "Deterministic first". +const AI_TEST_QUICK = { + suites: ["editor-context", "unsaved-buffers", "self-sufficiency", "bug-fixing"], + tests: ["UB-1", "EC-5", "EC-2", "SS-4", "EC-1", "UB-2", "SS-1", "BF-1"] +}; + +function _gitInfo(cwd) { + try { + const rev = execSync("git rev-parse --short HEAD", { cwd, stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); + const branch = execSync("git branch --show-current", { cwd, stdio: ["ignore", "pipe", "ignore"] }).toString().trim(); + const dirty = execSync("git status --porcelain", { cwd, stdio: ["ignore", "pipe", "ignore"] }).toString().trim() ? " (uncommitted changes)" : ""; + return `${rev} on ${branch}${dirty}`; + } catch (e) { + return "unknown"; + } +} + +function _claudeCliVersion() { + try { + return execSync("claude --version", { stdio: ["ignore", "pipe", "ignore"], timeout: 5000 }).toString().trim(); + } catch (e) { + return "unknown (read it from the panel's logs)"; + } +} + +// Two committed files only: baseline.md (the reference, edited by hand) and +// latest.md (overwritten every run — git history is the archive). Anything +// else in the folder is listed but not special. +const AI_TEST_REPORT_LATEST = "latest.md"; +const AI_TEST_REPORT_BASELINE = "baseline.md"; +function _listReports() { + const dir = path.join(AI_TESTS_DIR, "reports"); + if (!fs.existsSync(dir)) { return []; } + return fs.readdirSync(dir).filter(f => f.endsWith(".md")).sort(); +} +// The committed version of a report file, for `against: "previous"`. +function _gitHeadVersion(fileName) { + const proDir = path.join(REPO_ROOT, "src", "extensionsIntegrated", "phoenix-pro"); + const rel = path.posix.join("unit-tests", "ai_model_tests", "reports", fileName); + try { + return execSync(`git show HEAD:${rel}`, { cwd: proDir, stdio: ["ignore", "pipe", "ignore"] }).toString(); + } catch (e) { + return null; + } +} + +function _countTests(markdown) { + return (markdown.match(/^## [A-Z]{2}-\d+/gm) || []).length; +} + +function _todayStamp() { + return new Date().toISOString().slice(0, 10); +} + +const AI_TEST_RUNS_DIR = path.join(AI_TESTS_DIR, "reports", "runs"); + +function _runPath(runId) { + return path.join(AI_TEST_RUNS_DIR, runId.replace(/[^a-zA-Z0-9-]+/g, "-") + ".json"); +} +function _saveRun(run) { + fs.mkdirSync(AI_TEST_RUNS_DIR, { recursive: true }); + fs.writeFileSync(_runPath(run.runId), JSON.stringify(run, null, 2) + "\n", "utf8"); +} +function _loadRun(runId) { + const p = _runPath(runId); + return fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : null; +} +function _listRuns() { + if (!fs.existsSync(AI_TEST_RUNS_DIR)) { return []; } + return fs.readdirSync(AI_TEST_RUNS_DIR).filter(f => f.endsWith(".json")).map(f => f.slice(0, -5)).sort().reverse(); +} +function _countResults(results) { + const c = { PASS: 0, FAIL: 0, FLAKY: 0, BLOCKED: 0, good: 0, acceptable: 0, poor: 0 }; + for (const r of results) { + if (c[r.invariants] !== undefined) { c[r.invariants]++; } + if (c[r.judgement] !== undefined) { c[r.judgement]++; } + } + return c; +} +// Every "## XX-n — title" heading across the suite files, keyed by id. +function _loadTestCatalog() { + const catalog = {}; + for (const [suite, file] of Object.entries(AI_TEST_SUITES)) { + const full = path.join(AI_TESTS_DIR, file); + if (!fs.existsSync(full)) { continue; } + for (const m of fs.readFileSync(full, "utf8").matchAll(/^## ([A-Z]{2}-\d+)\s*—\s*(.+)$/gm)) { + catalog[m[1]] = { suite, file, title: m[2].trim() }; + } + } + return catalog; +} +// Rows of a report's "## Results" table, keyed by test id. Tolerates the +// baseline's range rows ("EC-2..EC-6") by keying them as written. +function _parseResults(markdown) { + const out = {}; + const sec = markdown.split(/^## Results/m)[1]; + if (!sec) { return out; } + for (const line of sec.split("\n")) { + if (!line.startsWith("|") || /^\|\s*-/.test(line) || /^\|\s*Suite/.test(line)) { continue; } + const cells = line.split("|").slice(1, -1).map(c => c.trim()); + if (cells.length < 4) { continue; } + const id = cells[1]; + if (!/^[A-Z]{2}-\d/.test(id)) { continue; } + const num = v => { const m = String(v || "").match(/[\d.]+/); return m ? parseFloat(m[0]) : null; }; + // Result cell is "PASS · poor" (or older reports: separate Invariants + // and Judgement columns). Detect which by whether column 3 looks like a + // judgement word. + const parts = (cells[2] || "").split(/\s*[·/]\s*/); + const separate = /^(good|acceptable|poor|n\/a|—|-)$/i.test(cells[3] || ""); + const inv = parts[0].split(/\s/)[0].toUpperCase(); + const jud = (separate ? cells[3] : (parts[1] || "")).split(/\s/)[0].toLowerCase(); + const off = separate ? 4 : 3; + out[id] = { inv, jud, tools: num(cells[off]), turns: num(cells[off + 1]), timeS: num(cells[off + 2]), cost: num(cells[off + 3]) }; + } + return out; +} + export function registerTools(server, processManager, wsControlServer, phoenixDesktopPath) { server.tool( "start_phoenix", @@ -512,4 +656,313 @@ export function registerTools(server, processManager, wsControlServer, phoenixDe }; } ); + + server.tool( + "run_ai_test_suite", + "Start (or resume) the AI panel model tests. You (the calling session) are the runner and the judge: " + + "this installs the fixture, gathers the environment, opens a run record for progress tracking, and " + + "returns the runner briefing plus the test documents to follow step by step. Scope: suite = 'quick' " + + "(default, ~4 model runs + free checks) | 'all' (~20 model runs) | one of " + + Object.keys(AI_TEST_SUITES).join(", ") + "; or tests = explicit IDs like ['EC-1','UB-2'] to run only " + + "those. resumeRunId continues an earlier run's remaining tests. After every test call ai_test_progress; " + + "when done (or told to stop) call save_ai_test_report and tell the user where it is.", + { + suite: z.string().optional().describe("quick | all | " + Object.keys(AI_TEST_SUITES).join(" | ")), + tests: z.array(z.string()).optional().describe("Explicit test IDs to run, e.g. [\"EC-1\",\"UB-2\"]. Overrides suite."), + resumeRunId: z.string().optional().describe("Run id from a previous briefing; runs only its remaining tests") + }, + async ({ suite, tests, resumeRunId }) => { + if (!fs.existsSync(path.join(AI_TESTS_DIR, "model_tests.md"))) { + return { content: [{ type: "text", text: "Test suite not found at " + AI_TESTS_DIR + + ". Is phoenix-pro checked out inside this phoenix repo?" }], isError: true }; + } + const catalog = _loadTestCatalog(); // { "EC-1": { suite, file, title }, ... } + + let which, planned, run; + if (resumeRunId) { + run = _loadRun(resumeRunId); + if (!run) { + return { content: [{ type: "text", text: "No run record '" + resumeRunId + "'. Known runs: " + + _listRuns().join(", ") }], isError: true }; + } + const done = new Set(run.results.map(r => r.test)); + planned = run.planned.filter(t => !done.has(t)); + which = run.scope; + run.status = "running"; + run.resumedAt = new Date().toISOString(); + } else if (tests && tests.length) { + const unknown = tests.filter(t => !catalog[t.toUpperCase()]); + if (unknown.length) { + return { content: [{ type: "text", text: "Unknown test id(s): " + unknown.join(", ") + + ". Known: " + Object.keys(catalog).join(", ") }], isError: true }; + } + planned = tests.map(t => t.toUpperCase()); + which = "selected"; + } else { + which = (suite || "quick").trim().toLowerCase(); + if (which !== "quick" && which !== "all" && !AI_TEST_SUITES[which]) { + return { content: [{ type: "text", text: "Unknown suite '" + which + "'. Use quick, all, or one of: " + + Object.keys(AI_TEST_SUITES).join(", ") }], isError: true }; + } + planned = which === "quick" ? AI_TEST_QUICK.tests.slice() + : Object.keys(catalog).filter(id => which === "all" || catalog[id].suite === which); + } + if (!run) { + run = { + runId: _todayStamp() + "-" + new Date().toTimeString().slice(0, 5).replace(":", "") + "-" + which, + scope: which, planned, results: [], status: "running", + startedAt: new Date().toISOString(), env: {} + }; + } + + // H0: install the fixture. Deterministic, so do it here rather than ask the runner. + const fixtureSrc = path.join(AI_TESTS_DIR, "fixtures", "taskboard"); + fs.mkdirSync(AI_TESTS_FIXTURE_DEST, { recursive: true }); + for (const f of fs.readdirSync(fixtureSrc)) { + fs.copyFileSync(path.join(fixtureSrc, f), path.join(AI_TESTS_FIXTURE_DEST, f)); + } + + const instances = wsControlServer.getConnectedInstances(); + const electron = instances.filter(i => i.startsWith("phoenix-electron-")); + const proDir = path.join(REPO_ROOT, "src", "extensionsIntegrated", "phoenix-pro"); + run.env = { phoenix: _gitInfo(REPO_ROOT), phoenixPro: _gitInfo(proDir), claudeCli: _claudeCliVersion(), instance: electron[0] || null }; + _saveRun(run); + + const suiteFiles = [...new Set(planned.map(id => catalog[id].file))]; + const docs = suiteFiles.map(file => ({ file, md: fs.readFileSync(path.join(AI_TESTS_DIR, file), "utf8") })); + const index = fs.readFileSync(path.join(AI_TESTS_DIR, "model_tests.md"), "utf8"); + const reports = _listReports(); + const baseline = reports.includes(AI_TEST_REPORT_BASELINE) ? AI_TEST_REPORT_BASELINE : null; + const reportName = AI_TEST_REPORT_LATEST; + + const briefing = [ + "# AI model tests — runner briefing", + "", + "You are the runner and the judge. Follow the documents below exactly. Do every step as written,", + "check every invariant, apply every judgement rubric, and write one or two sentences of reasoning", + "per test. Run the deterministic checks first, and stop if one fails.", + "", + "## Progress, stopping, and the user", + `- Run id: **${run.runId}**. After EVERY test, call ai_test_progress({ runId, test, invariants, judgement,`, + " tools, turns, timeMs, cost, reasoning }). It returns done/total and what remains.", + "- After every test, tell the user one line: ` — n/total done`.", + `- The run record the user can open at any time: ${_runPath(run.runId)}`, + "- If the user says stop: finish the step you are in, call ai_test_progress({ runId, stop: true }),", + " then save_ai_test_report with what you have — the report is marked partial and lists what was not run.", + " It can be resumed later with run_ai_test_suite({ resumeRunId }).", + "", + "## What will run", + `- Scope: **${which}** — ${planned.length} test(s)${resumeRunId ? " remaining" : ""}: ${planned.join(", ")}`, + ...suiteFiles.map(f => ` - ${f}`), + which === "quick" ? "- Quick means ONLY the listed tests. Skip everything else in those documents." : "", + which === "selected" ? "- Selected means ONLY the listed tests. Skip everything else in those documents." : "", + "", + "## Environment (put this at the top of the report)", + "- Runner model: ", + "- Model under test: ", + `- phoenix: ${run.env.phoenix}`, + `- phoenix-pro: ${run.env.phoenixPro}`, + `- Claude CLI: ${run.env.claudeCli}`, + `- Connected instances: ${instances.length ? instances.join(", ") : "none — start Phoenix first"}`, + electron.length ? `- Use instance: ${electron[0]}` : "- No phoenix-electron-* instance is connected. Ask the user to open the desktop app, then call get_phoenix_status.", + "", + "## Where things are", + `- Fixture installed (H0 done for you): ${AI_TESTS_FIXTURE_DEST}`, + `- In the documents, replace with: ${AI_TESTS_FIXTURE_DEST}`, + `- Transcript folder for H10: ~/.claude/projects/${AI_TESTS_FIXTURE_DEST.replace(/[\/ ]/g, "-")}`, + `- Test documents: ${AI_TESTS_DIR}`, + `- Reports folder: ${path.join(AI_TESTS_DIR, "reports")}`, + `- Compare against: ${baseline ? path.join(AI_TESTS_DIR, "reports", baseline) : "no baseline found — this run becomes the baseline"}`, + `- Reports in the folder: ${reports.length ? reports.join(", ") : "none"} (earlier runs live in git history of ${reportName})`, + `- Your report will be saved as: ${reportName} (overwrites the previous run) — call save_ai_test_report({ content, runId: "${run.runId}" }).`, + " Then call compare_ai_test_reports({}) to diff it against the baseline — or { against: \"previous\" } for the last committed run — and relay the result.", + "", + "---", + "", + "# Document 1 of " + (docs.length + 1) + ": model_tests.md (rules, harness, report format)", + "", + index, + ...docs.flatMap((d, i) => ["", "---", "", `# Document ${i + 2} of ${docs.length + 1}: ${d.file}`, "", d.md]) + ].filter(l => l !== null).join("\n"); + + return { content: [{ type: "text", text: briefing }] }; + } + ); + + server.tool( + "ai_test_progress", + "Record one test's result during an AI model test run, or read a run's progress, or stop it. With test + " + + "result fields: appends the result and returns done/total/remaining. With only runId: returns current " + + "progress (use this to answer 'how far along is it?'). With stop: true: marks the run stopped and returns " + + "what was not run. With no runId at all: lists known runs.", + { + runId: z.string().optional().describe("Run id from the briefing"), + test: z.string().optional().describe("Test id, e.g. EC-1"), + invariants: z.enum(["PASS", "FAIL", "FLAKY", "BLOCKED"]).optional(), + judgement: z.enum(["good", "acceptable", "poor", "n/a"]).optional(), + tools: z.number().optional(), turns: z.number().optional(), + timeMs: z.number().optional(), cost: z.number().optional(), + reasoning: z.string().optional().describe("One or two sentences: what the AI did and why that verdict"), + stop: z.boolean().optional().describe("Mark the run stopped (user asked to stop)") + }, + async (args) => { + if (!args.runId) { + const runs = _listRuns().map(id => { const r = _loadRun(id); return `${id}: ${r.status}, ${r.results.length}/${r.planned.length}`; }); + return { content: [{ type: "text", text: runs.length ? "Known runs:\n" + runs.join("\n") : "No runs recorded." }] }; + } + const run = _loadRun(args.runId); + if (!run) { + return { content: [{ type: "text", text: "No run record '" + args.runId + "'. Known: " + _listRuns().join(", ") }], isError: true }; + } + if (args.test) { + const id = args.test.toUpperCase(); + run.results = run.results.filter(r => r.test !== id); // a rerun replaces + run.results.push({ + test: id, invariants: args.invariants || "BLOCKED", judgement: args.judgement || "n/a", + tools: args.tools, turns: args.turns, timeMs: args.timeMs, cost: args.cost, + reasoning: args.reasoning || "", at: new Date().toISOString() + }); + } + if (args.stop) { + run.status = "stopped"; + run.stoppedAt = new Date().toISOString(); + } else if (run.results.length >= run.planned.length && run.planned.every(t => run.results.some(r => r.test === t))) { + run.status = "complete"; + run.completedAt = new Date().toISOString(); + } + _saveRun(run); + const done = run.planned.filter(t => run.results.some(r => r.test === t)); + const remaining = run.planned.filter(t => !done.includes(t)); + const counts = _countResults(run.results); + return { content: [{ type: "text", text: JSON.stringify({ + runId: run.runId, status: run.status, done: done.length, total: run.planned.length, + remaining, counts, record: _runPath(run.runId), + progressLine: `${done.length}/${run.planned.length} done · PASS ${counts.PASS} FAIL ${counts.FAIL} FLAKY ${counts.FLAKY} BLOCKED ${counts.BLOCKED}` + + (remaining.length ? ` · next: ${remaining[0]}` : " · all done") + }, null, 2) }] }; + } + ); + + server.tool( + "save_ai_test_report", + "File an AI model test report as reports/latest.md, overwriting the previous run (git history keeps " + + "earlier runs; baseline.md is never touched). If runId is given and the run was stopped, the report is " + + "marked partial and the unrun tests are appended. Returns the saved path and the baseline to compare " + + "against — relay the path to the user.", + { + content: z.string().describe("The full report markdown, following the skeleton in model_tests.md"), + runId: z.string().optional().describe("Run id from the briefing; links the report to its progress record"), + label: z.string().optional().describe("Ignored for the filename (always latest.md); kept for compatibility") + }, + async ({ content, runId }) => { + const dir = path.join(AI_TESTS_DIR, "reports"); + fs.mkdirSync(dir, { recursive: true }); + let body = content.endsWith("\n") ? content : content + "\n"; + const run = runId ? _loadRun(runId) : null; + let partial = false; + if (run) { + const notRun = run.planned.filter(t => !run.results.some(r => r.test === t)); + if (run.status === "stopped" || notRun.length) { + partial = true; + body += `\n## Partial run\nStopped ${run.stoppedAt || "before completion"}. Not run: ${notRun.join(", ") || "none"}.\n` + + `Resume with run_ai_test_suite({ resumeRunId: "${run.runId}" }).\n`; + } + } + const full = path.join(dir, AI_TEST_REPORT_LATEST); + const hadPrevious = fs.existsSync(full); + fs.writeFileSync(full, body, "utf8"); + if (run) { + // A finished run's progress record has done its job; only a + // stopped (resumable) run keeps its file. Keeps runs/ from piling up. + if (partial) { run.report = full; _saveRun(run); } + else if (fs.existsSync(_runPath(run.runId))) { fs.unlinkSync(_runPath(run.runId)); } + } + const hasBaseline = fs.existsSync(path.join(dir, AI_TEST_REPORT_BASELINE)); + return { content: [{ type: "text", text: JSON.stringify({ + saved: full, + partial, + overwrotePrevious: hadPrevious, + compareAgainst: hasBaseline ? path.join(dir, AI_TEST_REPORT_BASELINE) : null, + tellTheUser: "Report saved to " + full + (partial ? " (partial run)" : "") + + (hadPrevious ? "; the previous run is in git history" : "") + + (hasBaseline ? "; compare with compare_ai_test_reports({}) or { against: \"previous\" }" : "") + "." + }, null, 2) }] }; + } + ); + + server.tool( + "compare_ai_test_reports", + "Diff two AI model test reports test by test: invariant changes (REGRESSION / fixed), judgement changes " + + "(quality drop / improved), and tool/turn/time deltas with the thresholds from model_tests.md (tools " + + ">1.5x or time >2x = 'slower'). Defaults: report = latest.md, against = baseline.md. against: 'previous' " + + "compares latest.md with its last committed version (git HEAD). File names or absolute paths also accepted.", + { + report: z.string().optional().describe("Report to evaluate. Default: latest.md"), + against: z.string().optional().describe("Reference: 'baseline' (default), 'previous' (git HEAD of latest.md), a file name, or a path") + }, + async ({ report, against }) => { + const dir = path.join(AI_TESTS_DIR, "reports"); + const reports = _listReports(); + const resolve = (n, fallback) => { + const pick = n || fallback; + if (!pick) { return null; } + const full = path.isAbsolute(pick) ? pick : path.join(dir, pick); + return fs.existsSync(full) ? full : null; + }; + const a = resolve(report, AI_TEST_REPORT_LATEST); + let b, bLabel; + if ((against || "").toLowerCase() === "previous") { + const prev = _gitHeadVersion(AI_TEST_REPORT_LATEST); + if (!prev) { + return { content: [{ type: "text", text: "No committed version of " + AI_TEST_REPORT_LATEST + + " in git HEAD to compare against." }], isError: true }; + } + b = path.join(os.tmpdir(), "ai-model-tests-previous-latest.md"); fs.writeFileSync(b, prev, "utf8"); + bLabel = AI_TEST_REPORT_LATEST + " @ git HEAD"; + } else { + b = resolve((against || "").toLowerCase() === "baseline" ? null : against, AI_TEST_REPORT_BASELINE); + bLabel = b ? path.basename(b) : null; + } + if (!a || !b) { + return { content: [{ type: "text", text: "Need two reports. Have: " + (reports.join(", ") || "none") + + (a ? "" : " — report not found") + (b ? "" : " — reference not found") }], isError: true }; + } + const A = _parseResults(fs.readFileSync(a, "utf8")), B = _parseResults(fs.readFileSync(b, "utf8")); + const ids = [...new Set([...Object.keys(B), ...Object.keys(A)])].sort(); + const rows = [], flags = { regression: [], fixed: [], qualityDrop: [], improved: [], slower: [], faster: [], onlyInReport: [], onlyInReference: [] }; + const rank = { good: 2, acceptable: 1, poor: 0 }; + for (const id of ids) { + const x = A[id], y = B[id]; + if (!x) { flags.onlyInReference.push(id); rows.push(`| ${id} | — | ${y.inv}/${y.jud} | not in report |`); continue; } + if (!y) { flags.onlyInReport.push(id); rows.push(`| ${id} | ${x.inv}/${x.jud} | — | new |`); continue; } + const notes = []; + if (y.inv === "PASS" && x.inv === "FAIL") { flags.regression.push(id); notes.push("**REGRESSION**"); } + if (y.inv === "FAIL" && x.inv === "PASS") { flags.fixed.push(id); notes.push("fixed"); } + if (rank[x.jud] !== undefined && rank[y.jud] !== undefined) { + if (rank[x.jud] < rank[y.jud]) { flags.qualityDrop.push(id); notes.push("quality drop"); } + if (rank[x.jud] > rank[y.jud]) { flags.improved.push(id); notes.push("improved"); } + } + if (x.tools && y.tools && x.tools > 1.5 * y.tools) { flags.slower.push(id); notes.push(`tools ${y.tools}→${x.tools}`); } + if (x.timeS && y.timeS && x.timeS > 2 * y.timeS) { if (!flags.slower.includes(id)) { flags.slower.push(id); } notes.push(`time ${y.timeS}s→${x.timeS}s`); } + if (x.timeS && y.timeS && x.timeS < 0.5 * y.timeS && x.inv === "PASS") { flags.faster.push(id); notes.push(`faster ${y.timeS}s→${x.timeS}s`); } + rows.push(`| ${id} | ${x.inv}/${x.jud} | ${y.inv}/${y.jud} | ${notes.join(", ") || "same"} |`); + } + const out = [ + `# Comparison: ${path.basename(a)} vs ${bLabel}`, + "", + `Regressions: ${flags.regression.length ? flags.regression.join(", ") : "none"}`, + `Quality drops: ${flags.qualityDrop.length ? flags.qualityDrop.join(", ") : "none"}`, + `Slower (tools >1.5x or time >2x): ${flags.slower.length ? flags.slower.join(", ") : "none"}`, + `Fixed: ${flags.fixed.join(", ") || "none"} · Improved judgement: ${flags.improved.join(", ") || "none"} · Faster: ${flags.faster.join(", ") || "none"}`, + flags.onlyInReport.length ? `Only in report (no reference): ${flags.onlyInReport.join(", ")}` : "", + flags.onlyInReference.length ? `Not run this time: ${flags.onlyInReference.join(", ")}` : "", + "", + "| Test | report (inv/judg) | reference (inv/judg) | change |", + "| --- | --- | --- | --- |", + ...rows + ].filter(Boolean).join("\n"); + return { content: [{ type: "text", text: out }] }; + } + ); + } diff --git a/src-node/claude-code-agent.js b/src-node/claude-code-agent.js index c8bf7a6140..82aea2933d 100644 --- a/src-node/claude-code-agent.js +++ b/src-node/claude-code-agent.js @@ -315,6 +315,43 @@ function _formatAnswers(answer) { return answerText.trim(); } +/** + * Render the editor context the panel sent into the line prepended to the + * prompt. The panel assembles it because that is where the data and the + * user's context chips already live; this only turns it into prose. Returns + * "" when the panel sent nothing, i.e. the user dismissed those chips. + */ +function _buildEditorContextLine(ctx) { + if (!ctx || (!ctx.activeFile && !ctx.livePreviewFile)) { + return ""; + } + const parts = ["Editor state (auto-supplied, no tool call needed):"]; + if (ctx.activeFile) { + parts.push("the user is editing " + ctx.activeFile + "."); + if (ctx.unsaved) { + // Read and Edit are buffer-safe here (the agent flushes the buffer + // first); only Grep sees stale disk. Saying "stale on disk" without that + // steered the model off Edit onto the editor API — no edit card, no undo. + parts.push("Unsaved (Read and Edit see the unsaved text as normal; Grep does not, so " + + "use searchEditorBuffers to search these): " + ctx.unsaved + "."); + } + } + if (ctx.livePreviewFile) { + parts.push(ctx.livePreviewFile === ctx.activeFile + ? "The live preview is showing that same file." + : "The live preview is showing " + ctx.livePreviewFile + "."); + } + // Say plainly when the lists are complete. Left merely to infer it, the + // agent calls getEditorState to check — the exact lookup this line is + // here to save. + parts.push(ctx.truncated + ? "Trust this over searching for it yourself; call getEditorState for the names cut " + + "from a list, or if you need the cursor, the selection or a fresher view." + : "That is the complete set. Trust it over searching or double-checking; call " + + "getEditorState only if you need the cursor, the selection or a fresher view."); + return parts.join(" "); +} + /** * Detect whether a PostToolUse `tool_response` represents an error result. * Used to suppress diff-card painting when the SDK's native Edit/Write itself @@ -855,7 +892,8 @@ exports.checkAvailability = async function (opts) { * aiProgress, aiTextStream, aiToolEdit, aiError, aiComplete */ exports.sendPrompt = async function (params) { - const { prompt, projectPath, sessionAction, model, locale, selectionContext, images, envOverrides, permissionMode, additionalDirectories } = params; + const { prompt, projectPath, sessionAction, model, locale, selectionContext, editorContext, + images, envOverrides, permissionMode, additionalDirectories } = params; const requestId = Date.now().toString(36) + Math.random().toString(36).slice(2, 7); // Handle session @@ -874,6 +912,12 @@ exports.sendPrompt = async function (params) { currentAbortController = new AbortController(); + // Prepend what the user is looking at. The panel knows the active file, + // the unsaved buffers and the live preview target for certain, so stating + // them costs one line and removes the reason to go hunting: without it + // the model opens by grepping the project for a file already on screen. + const editorContextLine = _buildEditorContextLine(editorContext); + // Prepend selection context to the prompt if available let enrichedPrompt = prompt; if (selectionContext) { @@ -896,6 +940,9 @@ exports.sendPrompt = async function (params) { " to read the selected content if needed." + previewSnippet + "\n" + prompt; } } + if (editorContextLine) { + enrichedPrompt = editorContextLine + "\n\n" + enrichedPrompt; + } // Run the query asynchronously — don't await here so we return requestId immediately _runQuery(requestId, enrichedPrompt, projectPath, model, currentAbortController.signal, locale, images, envOverrides, permissionMode, additionalDirectories) @@ -1378,6 +1425,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale, "WebFetch", "WebSearch", "EnterPlanMode", "ExitPlanMode", "mcp__phoenix-editor__getEditorState", + "mcp__phoenix-editor__searchEditorBuffers", "mcp__phoenix-editor__takeScreenshot", "mcp__phoenix-editor__execJsInLivePreview", "mcp__phoenix-editor__execJsInEditor", @@ -1399,6 +1447,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale, " files to answer questions. Do not modify files.", tools: ["Read", "Glob", "Grep", "mcp__phoenix-editor__getEditorState", + "mcp__phoenix-editor__searchEditorBuffers", "mcp__phoenix-editor__takeScreenshot", "mcp__phoenix-editor__execJsInLivePreview", "mcp__phoenix-editor__editorDocs"] @@ -1411,6 +1460,7 @@ async function _runQuery(requestId, prompt, projectPath, model, signal, locale, " only for new files.", tools: ["Read", "Edit", "Write", "Glob", "Grep", "mcp__phoenix-editor__getEditorState", + "mcp__phoenix-editor__searchEditorBuffers", "mcp__phoenix-editor__takeScreenshot", "mcp__phoenix-editor__execJsInLivePreview", "mcp__phoenix-editor__execJsInEditor", diff --git a/src-node/mcp-editor-tools.js b/src-node/mcp-editor-tools.js index 287c865a4a..cca2a8d21f 100644 --- a/src-node/mcp-editor-tools.js +++ b/src-node/mcp-editor-tools.js @@ -54,7 +54,8 @@ const EXEC_PEER_TIMEOUT_MS = { getEditorState: 5000, takeScreenshot: 15000, controlEditor: 5000, - resizeLivePreview: 5000 + resizeLivePreview: 5000, + searchEditorBuffers: 3000 }; // Floor for caller-provided timeouts (e.g. execJsInLivePreview's @@ -154,7 +155,57 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } return _maybeAppendHint(result, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "which file the user has open in Phoenix Code editor, plus cursor, selection, and what the live preview (an embedded browser rendering their HTML or Markdown) is showing" + } + ); + + const searchEditorBuffersTool = sdkModule.tool( + "searchEditorBuffers", + "Regex search over the UNSAVED open files only — the ones the editor-state line at the top of " + + "the prompt lists as unsaved. Those are the only files where Grep is wrong: Grep reads disk, and " + + "disk is stale for a buffer the user has edited but not saved. Use Grep for everything else; it " + + "is faster and covers the whole project. Only call this when the editor-state line names unsaved " + + "files. Returns matches {file, line, text}, searchedFiles (what this actually covered) and truncated.", + { + pattern: z.string().describe("Regex (default) or literal text to find"), + isRegex: z.boolean().optional().describe("false to match the pattern literally. Default true"), + caseSensitive: z.boolean().optional().describe("Default false"), + fileGlob: z.string().optional().describe("Limit to matching files, e.g. *.css"), + maxResults: z.number().optional().describe("Cap on matches returned. Default 50, max 200") + }, + async function (args) { + let result; + try { + const found = await _execPeerWithTimeout(nodeConnector, "searchEditorBuffers", + args || {}, "searchEditorBuffers"); + let text; + if (found && found.error) { + text = JSON.stringify(found); + } else if (!found || !found.searchedFiles || !found.searchedFiles.length) { + text = "No unsaved files, so nothing in the editor differs from disk. Use Grep — " + + "it is authoritative for the whole project right now."; + } else { + text = JSON.stringify(found) + + "\n\nThis searched ONLY the unsaved files in searchedFiles. Every other file " + + "matches disk — use Grep for the rest of the project."; + } + result = { content: [{ type: "text", text: text }] }; + } catch (err) { + result = { + content: [{ type: "text", text: "Error searching unsaved files: " + err.message }], + isError: true + }; + } + return _maybeAppendHint(result, hasClarification); + }, + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "search the unsaved editor buffers, where Grep would see stale disk content" + } ); const takeScreenshotTool = sdkModule.tool( @@ -219,7 +270,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "screenshot the user's Phoenix Code editor app window, or the page rendered in their live preview browser" + } ); const execJsInLivePreviewTool = sdkModule.tool( @@ -265,7 +320,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "run JS in the user's live preview browser to inspect the rendered page's DOM, console or JS state" + } ); const controlEditorTool = sdkModule.tool( @@ -330,7 +389,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "open, close or switch files in Phoenix Code, toggle the live preview browser" + } ); const resizeLivePreviewTool = sdkModule.tool( @@ -364,7 +427,11 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) } return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + alwaysLoad: true, + searchHint: "resize the user's live preview browser viewport to check a responsive layout" + } ); const waitTool = sdkModule.tool( @@ -383,7 +450,10 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + searchHint: "pause before re-checking the user's live preview browser" + } ); const execJsInEditorTool = sdkModule.tool( @@ -457,6 +527,9 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; } return _maybeAppendHint(toolResult, hasClarification); + }, + { + searchHint: "run JS against Phoenix Code's own editor API, not the page in its live preview" } ); @@ -527,6 +600,9 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; } return _maybeAppendHint(toolResult, hasClarification); + }, + { + searchHint: "read or change the user's Phoenix Code editor preferences" } ); @@ -574,7 +650,10 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) }; return _maybeAppendHint(toolResult, hasClarification); }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + searchHint: "look up Phoenix Code editor feature or API documentation" + } ); const getUserClarificationTool = sdkModule.tool( @@ -607,12 +686,15 @@ function createEditorMcpServer(sdkModule, nodeConnector, clarificationAccessors) content: [{ type: "text", text: "No clarification queued." }] }; }, - { annotations: { readOnlyHint: true } } + { + annotations: { readOnlyHint: true }, + searchHint: "read a follow-up the user typed into this conversation while you were still working" + } ); return sdkModule.createSdkMcpServer({ name: "phoenix-editor", - tools: [getEditorStateTool, takeScreenshotTool, execJsInLivePreviewTool, + tools: [getEditorStateTool, searchEditorBuffersTool, takeScreenshotTool, execJsInLivePreviewTool, execJsInEditorTool, editorPreferencesTool, editorDocsTool, controlEditorTool, resizeLivePreviewTool, waitTool, getUserClarificationTool] }); diff --git a/src/nls/root/strings.js b/src/nls/root/strings.js index cd283fb661..a45d9cbae1 100644 --- a/src/nls/root/strings.js +++ b/src/nls/root/strings.js @@ -2659,6 +2659,8 @@ define({ "AI_CHAT_TOOL_SCREENSHOT_LIVE_PREVIEW": "live preview", "AI_CHAT_TOOL_SCREENSHOT_FULL_EDITOR": "the full editor", "AI_CHAT_TOOL_LIVE_PREVIEW_JS": "Inspecting preview", + "AI_CHAT_TOOL_SEARCH_UNSAVED": "Search unsaved files", + "AI_CHAT_TOOL_SEARCH_UNSAVED_FOR": "Search unsaved files: {0}", "AI_CHAT_TOOL_EDITOR_JS": "Inspecting editor", "AI_CHAT_TOOL_EDITOR_PREFERENCES": "Editor preferences", "AI_CHAT_TOOL_EDITOR_DOCS": "Editor docs", diff --git a/tracking-repos.json b/tracking-repos.json index fbcab96286..283e04c7bf 100644 --- a/tracking-repos.json +++ b/tracking-repos.json @@ -1,5 +1,5 @@ { "phoenixPro": { - "commitID": "2f0558b7cfccd51d8b12966f2bc7ef73a02dbbd6" + "commitID": "033ca7b4524c0e01a5d412893f5aee99a164a5e5" } }