diff --git a/src/cli/index.ts b/src/cli/index.ts index 1107382..5e679ac 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,6 +1,8 @@ #!/usr/bin/env node import { readFile, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; +import { resolve } from "node:path"; import { parseArgs, strFlag, boolFlag, numFlag, type ParsedArgs } from "./args.js"; import { loadSuite } from "../suite.js"; import { runSuite } from "../runner.js"; @@ -109,7 +111,7 @@ async function loadResult(path: string): Promise { return JSON.parse(await readFile(path, "utf8")) as RunResult; } -async function cmdCompare(args: ParsedArgs): Promise { +export async function cmdCompare(args: ParsedArgs): Promise { const basePath = strFlag(args, "base"); if (!basePath) throw new Error("compare requires --base "); const baseline = await loadResult(basePath); @@ -146,8 +148,15 @@ async function cmdCompare(args: ParsedArgs): Promise { if (!ctx) { console.error("[evalgate] --comment set but no GitHub PR context found; skipping."); } else { - await upsertComment(ctx, md); - console.log(`[evalgate] posted report to ${ctx.owner}/${ctx.repo}#${ctx.prNumber}`); + try { + await upsertComment(ctx, md); + console.log(`[evalgate] posted report to ${ctx.owner}/${ctx.repo}#${ctx.prNumber}`); + } catch (err) { + // Reporting is a side-effect: a failure (e.g. read-only fork token, HTTP 403) + // must not turn a passing comparison into a failed gate. + const status = err instanceof Error ? err.message : String(err); + console.error(`[evalgate] warning: could not post PR comment; skipping. (${status})`); + } } } @@ -228,9 +237,16 @@ async function main(): Promise { } } -main() - .then((code) => process.exit(code)) - .catch((err) => { - console.error((err as Error).message); - process.exit(2); - }); +// Run the CLI only when this module is the entry point, so importing it in +// tests does not execute main() against the test runner's argv. +const invoked = process.argv[1] + ? import.meta.url === pathToFileURL(resolve(process.argv[1])).href + : false; +if (invoked) { + main() + .then((code) => process.exit(code)) + .catch((err) => { + console.error((err as Error).message); + process.exit(2); + }); +} diff --git a/tests/comment-failure.test.ts b/tests/comment-failure.test.ts new file mode 100644 index 0000000..4e995eb --- /dev/null +++ b/tests/comment-failure.test.ts @@ -0,0 +1,63 @@ +import { afterAll, describe, expect, it, vi } from "vitest"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { cmdCompare } from "../src/cli/index.js"; +import type { RunResult } from "../src/types.js"; + +function result(score: number): RunResult { + return { + version: "1", + suite: "s", + timestamp: "now", + score, + passed: true, + total: 1, + passedCount: 1, + latencyMs: 0, + costUsd: 0, + cases: [ + { + id: "c1", + model: "mock", + provider: "mock", + output: "", + latencyMs: 1, + costUsd: 0, + score, + passed: true, + scores: [], + }, + ], + }; +} + +const savedFetch = globalThis.fetch; +afterAll(() => { + globalThis.fetch = savedFetch; +}); + +describe("cmdCompare comment side-effect", () => { + it("exits 0 when the comment API returns 403 on a fork PR", async () => { + const dir = await mkdtemp(join(tmpdir(), "evalgate-")); + const base = join(dir, "base.json"); + const head = join(dir, "head.json"); + await writeFile(base, JSON.stringify(result(0.94))); + await writeFile(head, JSON.stringify(result(0.94))); + + vi.stubEnv("GITHUB_TOKEN", "t"); + vi.stubEnv("GITHUB_REPOSITORY", "owner/repo"); + vi.stubEnv("EVALGATE_PR", "1"); + // Simulate the read-only fork token: every comment write gets 403. + globalThis.fetch = vi.fn(async () => new Response("forbidden", { status: 403 })); + + const code = await cmdCompare({ + _: ["compare"], + flags: { base, head, comment: "true" }, + }); + + expect(code).toBe(0); + vi.unstubAllEnvs(); + await rm(dir, { recursive: true, force: true }); + }); +});