Add GHCP end-to-end evaluation harness

ts/packages/copilot-plugin-eval/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

ts/packages/copilot-plugin-eval/README.AUTOGEN.md

@@ -0,0 +1,43 @@
+<!-- Copyright (c) Microsoft Corporation. -->
+<!-- Licensed under the MIT License. -->
+
+<!-- AUTOGEN:DOCS:START -->
+
+<!-- AUTOGEN:DOCS:HASH:sha256=2d0e6b07a09267e83572600c217225a8dd52bc6018a5f6dd3dd4b5e2d0874171 -->
+<!-- AUTOGEN:DOCS:SOURCE: ./README.md (hand-written documentation; this file is the AI-generated companion) -->
+
+# @typeagent/copilot-plugin-eval — AI-generated documentation
+
+> 📝 **Placeholder documentation — not yet AI-authored.** Re-run `pnpm docs:generate:llm --package copilot-plugin-eval` to populate this file, or read [`./README.md`](./README.md) for the hand-written documentation in the meantime. The deterministic Reference section below is already populated.
+
+## Overview
+
+Initial end-to-end Copilot plugin evaluation harness
+
+## Reference
+
+> ⚙️ **Auto-generated, no AI involvement.** Built deterministically from `package.json`, `src/`, and the workspace dependency graph at the commit recorded in the staleness footer at the end of this file. Hand edits to this file will be overwritten on the next run.
+
+### Entry points
+
+_No public exports declared in `package.json`._
+
+### Dependencies
+
+Workspace:
+
+- [@typeagent/copilot-plugin](../../packages/copilot-plugin/README.md)
+- [agent-dispatcher](../../packages/dispatcher/dispatcher/README.md)
+- [agent-server](../../packages/agentServer/server/README.md)
+
+External: _None at runtime._
+
+### Files of interest
+
+_No tracked source files under `./src/`._
+
+---
+
+_Auto-generated against commit `ba6a8b85088604d88c37360ec0bc092911ad38bc` on `2026-09-25T22:33:54.079Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/copilot-plugin-eval docs:verify-links` to spot-check._
+
+<!-- AUTOGEN:DOCS:END -->

ts/packages/copilot-plugin-eval/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@typeagent/copilot-plugin-eval",
+  "version": "0.0.1",
+  "description": "Initial end-to-end Copilot plugin evaluation harness",
+  "homepage": "https://github.com/microsoft/TypeAgent#readme",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/TypeAgent.git",
+    "directory": "ts/packages/copilot-plugin-eval"
+  },
+  "license": "MIT",
+  "author": "Microsoft",
+  "type": "module",
+  "scripts": {
+    "build": "node --check scripts/ghcp-eval.mjs && node --check scripts/ghcp-eval-preflight.mjs && node --check scripts/ghcp-credit-probe.mjs",
+    "prettier": "prettier --check . --ignore-path ../../.prettierignore",
+    "prettier:fix": "prettier --write . --ignore-path ../../.prettierignore",
+    "test": "npm run test:local",
+    "test:local": "node --test scripts/test/*.spec.mjs"
+  },
+  "devDependencies": {
+    "@github/copilot-sdk": "1.0.13",
+    "@modelcontextprotocol/sdk": "^1.26.0",
+    "@typeagent/copilot-plugin": "workspace:*",
+    "agent-dispatcher": "workspace:*",
+    "agent-server": "workspace:*",
+    "prettier": "^3.5.3"
+  }
+}

ts/packages/copilot-plugin-eval/scripts/ghcp-credit-probe.mjs

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import fs from "node:fs";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { CopilotClient, RuntimeConnection } from "@github/copilot-sdk";
+import { evalModel, validateEvalLedger } from "./ghcp-eval-config.mjs";
+import {
+    CopilotCreditBudget,
+    accountedNanoAiu,
+} from "../../dispatcher/dispatcher/dist/reasoning/copilotCreditBudget.js";
+
+const [cliPath, ledgerPath, outputDirectory] = process.argv.slice(2);
+if (!cliPath || !ledgerPath || !outputDirectory) {
+    throw new Error(
+        "Usage: node ghcp-credit-probe.mjs <copilot.exe> <ledger.json> <new-output-directory>",
+    );
+}
+const ledger = JSON.parse(fs.readFileSync(ledgerPath, "utf8"));
+accountedNanoAiu(ledger);
+validateEvalLedger(ledger);
+fs.mkdirSync(outputDirectory);
+const client = new CopilotClient({
+    mode: "empty",
+    baseDirectory: path.join(outputDirectory, "copilot"),
+    workingDirectory: outputDirectory,
+    connection: RuntimeConnection.forStdio({ path: cliPath }),
+    requestHandler: new CopilotCreditBudget(path.resolve(ledgerPath)),
+    useLoggedInUser: true,
+    logLevel: "error",
+});
+const result = {
+    kind: "credit_control_calibration_not_eval",
+    model: evalModel,
+    sessionId: randomUUID(),
+    status: "not_started",
+    usage: [],
+};
+let session;
+try {
+    await client.start();
+    session = await client.createSession({
+        sessionId: result.sessionId,
+        model: evalModel,
+        reasoningEffort: "low",
+        contextTier: "default",
+        sessionLimits: { maxAiCredits: 30 },
+        capi: { enableWebSocketResponses: false },
+        availableTools: [],
+        skipCustomInstructions: true,
+        onPermissionRequest: () => ({
+            kind: "denied-no-approval-rule-and-could-not-request-from-user",
+        }),
+    });
+    session.on("assistant.usage", (event) => {
+        result.usage.push({
+            model: event.data.model,
+            copilotUsage: event.data.copilotUsage,
+            inputTokens: event.data.inputTokens,
+            outputTokens: event.data.outputTokens,
+        });
+    });
+    result.status = "running";
+    await session.sendAndWait({ prompt: "Reply with exactly OK." }, 60_000);
+    result.status = "completed";
+} catch (error) {
+    result.status = "failed";
+    result.error = error instanceof Error ? error.message : String(error);
+    process.exitCode = 1;
+} finally {
+    if (session) await session.abort();
+    await client.stop();
+    fs.writeFileSync(
+        path.join(outputDirectory, "result.json"),
+        JSON.stringify(result, null, 2) + "\n",
+    );
+    process.stdout.write(JSON.stringify(result) + "\n");
+}

ts/packages/copilot-plugin-eval/scripts/ghcp-eval-config.mjs

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export const evalModel = "gpt-5.6-luna";
+
+export function validateEvalLedger(ledger) {
+    if (ledger?.model !== evalModel) {
+        throw new Error(
+            `Evaluation requires a ledger for ${evalModel}; preserve historical ledgers and reconcile a new model-specific run before admission`,
+        );
+    }
+}