fix(cli): warn instead of failing the gate when the PR comment cannot be posted

src/cli/index.ts

@@ -1,6 +1,8 @@
 #!/usr/bin/env node
 import { readFile, writeFile } from "node:fs/promises";
 import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+import { resolve } from "node:path";
 import { parseArgs, strFlag, boolFlag, numFlag, type ParsedArgs } from "./args.js";
 import { loadSuite } from "../suite.js";
 import { runSuite } from "../runner.js";
@@ -109,7 +111,7 @@ async function loadResult(path: string): Promise<RunResult> {
   return JSON.parse(await readFile(path, "utf8")) as RunResult;
 }
 
-async function cmdCompare(args: ParsedArgs): Promise<number> {
+export async function cmdCompare(args: ParsedArgs): Promise<number> {
   const basePath = strFlag(args, "base");
   if (!basePath) throw new Error("compare requires --base <baseline.json>");
   const baseline = await loadResult(basePath);
@@ -146,8 +148,15 @@ async function cmdCompare(args: ParsedArgs): Promise<number> {
     if (!ctx) {
       console.error("[evalgate] --comment set but no GitHub PR context found; skipping.");
     } else {
-      await upsertComment(ctx, md);
-      console.log(`[evalgate] posted report to ${ctx.owner}/${ctx.repo}#${ctx.prNumber}`);
+      try {
+        await upsertComment(ctx, md);
+        console.log(`[evalgate] posted report to ${ctx.owner}/${ctx.repo}#${ctx.prNumber}`);
+      } catch (err) {
+        // Reporting is a side-effect: a failure (e.g. read-only fork token, HTTP 403)
+        // must not turn a passing comparison into a failed gate.
+        const status = err instanceof Error ? err.message : String(err);
+        console.error(`[evalgate] warning: could not post PR comment; skipping. (${status})`);
+      }
     }
   }
 
@@ -228,9 +237,16 @@ async function main(): Promise<number> {
   }
 }
 
-main()
-  .then((code) => process.exit(code))
-  .catch((err) => {
-    console.error((err as Error).message);
-    process.exit(2);
-  });
+// Run the CLI only when this module is the entry point, so importing it in
+// tests does not execute main() against the test runner's argv.
+const invoked = process.argv[1]
+  ? import.meta.url === pathToFileURL(resolve(process.argv[1])).href
+  : false;
+if (invoked) {
+  main()
+    .then((code) => process.exit(code))
+    .catch((err) => {
+      console.error((err as Error).message);
+      process.exit(2);
+    });
+}

tests/comment-failure.test.ts

@@ -0,0 +1,63 @@
+import { afterAll, describe, expect, it, vi } from "vitest";
+import { mkdtemp, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { cmdCompare } from "../src/cli/index.js";
+import type { RunResult } from "../src/types.js";
+
+function result(score: number): RunResult {
+  return {
+    version: "1",
+    suite: "s",
+    timestamp: "now",
+    score,
+    passed: true,
+    total: 1,
+    passedCount: 1,
+    latencyMs: 0,
+    costUsd: 0,
+    cases: [
+      {
+        id: "c1",
+        model: "mock",
+        provider: "mock",
+        output: "",
+        latencyMs: 1,
+        costUsd: 0,
+        score,
+        passed: true,
+        scores: [],
+      },
+    ],
+  };
+}
+
+const savedFetch = globalThis.fetch;
+afterAll(() => {
+  globalThis.fetch = savedFetch;
+});
+
+describe("cmdCompare comment side-effect", () => {
+  it("exits 0 when the comment API returns 403 on a fork PR", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "evalgate-"));
+    const base = join(dir, "base.json");
+    const head = join(dir, "head.json");
+    await writeFile(base, JSON.stringify(result(0.94)));
+    await writeFile(head, JSON.stringify(result(0.94)));
+
+    vi.stubEnv("GITHUB_TOKEN", "t");
+    vi.stubEnv("GITHUB_REPOSITORY", "owner/repo");
+    vi.stubEnv("EVALGATE_PR", "1");
+    // Simulate the read-only fork token: every comment write gets 403.
+    globalThis.fetch = vi.fn(async () => new Response("forbidden", { status: 403 }));
+
+    const code = await cmdCompare({
+      _: ["compare"],
+      flags: { base, head, comment: "true" },
+    });
+
+    expect(code).toBe(0);
+    vi.unstubAllEnvs();
+    await rm(dir, { recursive: true, force: true });
+  });
+});