fix: an error path with nothing listening on it, five times over

.gitignore

@@ -20,3 +20,6 @@ runs/
 
 # Local Python environment for adapters that need one (gymnasium, ALE).
 .venv/
+
+# pnpm writes a store here when a container runs install against a mounted repo.
+.pnpm-store/

ale.test.mts

@@ -16,6 +16,7 @@
  * observation image channel, and worker teardown. Zero model spend.
  */
 import { strict as assert } from 'node:assert'
+import { resolve } from 'node:path'
 import { spawnSync } from 'node:child_process'
 import { createHash } from 'node:crypto'
 import { deriveContract } from './authoring'
@@ -121,7 +122,19 @@ const DEMO_COLLISIONS = [
     collisions: 56, freeTurns: 21, jointCollision: false, family: 521838526464,
   },
 ]
-const python = process.env.PLAYPROOF_PYTHON ?? 'python3'
+/**
+ * The interpreter, resolved against the CALLER's directory.
+ *
+ * A probe below runs with `cwd: tmpdir()` so it cannot import from the repo by
+ * accident, which means a relative `PLAYPROOF_PYTHON` never resolves and the
+ * gate reports the package missing when the real fault is the path. Measured:
+ * `PLAYPROOF_PYTHON=./.venv/bin/python` produced "stable-retro is not
+ * importable", advising an install of software that was already installed.
+ *
+ * A bare name like `python3` is left alone for PATH lookup.
+ */
+const rawPython = process.env.PLAYPROOF_PYTHON ?? 'python3'
+const python = rawPython.includes('/') ? resolve(rawPython) : rawPython
 
 /** The bundled ROM must be present, not just the package. */
 function pythonHasAle(): boolean {

drivers/persistent-cli.ts

@@ -236,9 +236,26 @@ export function createPersistentCliDriver(options: PersistentCliDriverOptions):
     const running = child
     child = null
     if (running !== null) {
-      running.stdout.removeAllListeners()
-      running.stderr.removeAllListeners()
-      running.stdin.removeAllListeners()
+      // Drop the DATA listeners and keep an error listener on every pipe.
+      //
+      // `removeAllListeners()` took the `error` handler with it, and the kill
+      // below makes a write to stdin fail. An in-flight write then completed
+      // with EPIPE on a socket that no longer had a listener, which Node turns
+      // into an unhandled `error` event and a dead process.
+      //
+      // MEASURED: three separate long studies died this way, each losing every
+      // finished cell, with a stack carrying no frame from this repository.
+      // The session's other three crashes were the same shape — an error path
+      // with nothing listening on it.
+      for (const pipe of [running.stdout, running.stderr, running.stdin]) {
+        pipe?.removeAllListeners('data')
+        pipe?.removeAllListeners('error')
+        pipe?.on('error', () => {
+          // A pipe to a process being killed is expected to fail. The session's
+          // outcome is already recorded by `end()`; this listener exists so the
+          // failure cannot escape as an unhandled event.
+        })
+      }
       running.kill('SIGKILL')
     }
   }

gymnasium.test.mts

@@ -12,6 +12,7 @@
  * About 10s, zero model spend.
  */
 import { strict as assert } from 'node:assert'
+import { resolve } from 'node:path'
 import { spawnSync } from 'node:child_process'
 import { tmpdir } from 'node:os'
 import { attestRun } from './attestation'
@@ -24,7 +25,19 @@ import { bundledReference, makeGymnasium, type Gymnasium, type GymState } from '
 
 const CARTPOLE = 'CartPole-v1'
 const FROZENLAKE = 'FrozenLake-v1'
-const python = process.env.PLAYPROOF_PYTHON ?? 'python3'
+/**
+ * The interpreter, resolved against the CALLER's directory.
+ *
+ * A probe below runs with `cwd: tmpdir()` so it cannot import from the repo by
+ * accident, which means a relative `PLAYPROOF_PYTHON` never resolves and the
+ * gate reports the package missing when the real fault is the path. Measured:
+ * `PLAYPROOF_PYTHON=./.venv/bin/python` produced "stable-retro is not
+ * importable", advising an install of software that was already installed.
+ *
+ * A bare name like `python3` is left alone for PATH lookup.
+ */
+const rawPython = process.env.PLAYPROOF_PYTHON ?? 'python3'
+const python = rawPython.includes('/') ? resolve(rawPython) : rawPython
 
 /**
  * Probe from a temporary directory, never from the repository root: Playproof's

matrix-run.ts

@@ -402,7 +402,15 @@ function lastOf(values: readonly number[]): number | null {
   return values.length === 0 ? null : values[values.length - 1]!
 }
 
-function blockedResult(cell: MatrixCell, reason: BlockedReason, detail: string, wallMs: number): CellResult {
+/**
+ * A cell that did not play, with the reason it did not.
+ *
+ * Exported so a runner can build the same row for a fault `runCell` could not
+ * foresee. One cell that throws must cost one cell, not the study around it,
+ * and it must land in the artifact looking like every other refusal rather than
+ * as a gap a reader has to notice.
+ */
+export function blockedResult(cell: MatrixCell, reason: BlockedReason, detail: string, wallMs: number): CellResult {
   return {
     ...identity(cell),
     status: 'blocked',

matrix.mts

@@ -14,7 +14,7 @@
 import { mkdir, readFile, writeFile } from 'node:fs/promises'
 import { dirname } from 'node:path'
 import { cellName, enumerateCells, parseMatrix } from './matrix'
-import { assertJoinable, effectiveArms, generalization, runCell, type CellResult } from './matrix-run'
+import { assertJoinable, blockedResult, effectiveArms, generalization, runCell, type CellResult } from './matrix-run'
 
 // SEVERAL definitions, pooled into one study.
 //
@@ -42,6 +42,50 @@ for (const path of definitionPaths) {
 const definitionPath = definitionPaths.join(' ')
 
 const rows: CellResult[] = []
+
+/**
+ * Write what has finished so far.
+ *
+ * Called after EVERY cell, not once at the end. A 42-cell study died at cell 3
+ * on an unhandled pipe error and left no artifact at all, so two cells that had
+ * each cost twenty minutes were lost after they had already succeeded. Work
+ * that is done should survive whatever happens to the work that is not.
+ *
+ * `partial` marks a file whose run has not finished, so a reader never mistakes
+ * an interrupted study for a complete one.
+ */
+async function persist(partial: boolean): Promise<void> {
+  const protocolsSeen = new Set(rows.map((row) => row.protocol))
+  const artifact = {
+    definition: definitionPath,
+    cells: rows.length,
+    expected: cells.length,
+    partial,
+    rows,
+    summary: partial || protocolsSeen.size !== 1
+      ? { transfer: null, arms: effectiveArms(rows), note: partial ? 'run did not finish' : 'several protocols' }
+      : { transfer: generalization(rows), arms: effectiveArms(rows) },
+  }
+  const text = `${JSON.stringify(artifact, null, 2)}\n`
+  if (outPath === undefined) return
+  await mkdir(dirname(outPath), { recursive: true })
+  await writeFile(outPath, text)
+}
+
+// A crash must not take finished work with it. What is on disk is written
+// first, then the failure is reported and the exit code still says it failed:
+// keeping the data is not the same as pretending the run succeeded.
+for (const signal of ['uncaughtException', 'unhandledRejection'] as const) {
+  process.on(signal, (error: unknown) => {
+    void persist(true).finally(() => {
+      console.error(`\nmatrix: ${signal} after ${rows.length} of ${cells.length} cells`)
+      console.error(error instanceof Error ? (error.stack ?? error.message) : String(error))
+      if (outPath !== undefined) console.error(`matrix: wrote ${rows.length} finished cells to ${outPath}`)
+      process.exit(1)
+    })
+  })
+}
+
 for (const [index, cell] of cells.entries()) {
   // A cell is announced BEFORE it runs, and says it is alive while it runs.
   //
@@ -63,6 +107,12 @@ for (const [index, cell] of cells.entries()) {
   let row: CellResult
   try {
     row = await runCell(cell)
+  } catch (error) {
+    // `runCell` returns a blocked row for everything it can foresee. This is
+    // for what it cannot: one cell that throws costs one cell, and the study
+    // keeps going and keeps what it has.
+    console.error(`[${index + 1}/${cells.length}] threw: ${(error as Error).message}`)
+    row = blockedResult(cell, 'episode-failed', (error as Error).message, Date.now() - cellStarted)
   } finally {
     clearInterval(heartbeat)
   }
@@ -95,14 +145,9 @@ const summary = protocols.size === 1
   : { transfer: null, arms: effectiveArms(rows), note: `${protocols.size} protocols: summarise each on its own` }
 if (protocols.size === 1) assertJoinable(rows)
 
-const artifact = { definition: definitionPath, cells: rows.length, rows, summary }
-if (outPath !== undefined) {
-  await mkdir(dirname(outPath), { recursive: true })
-  await writeFile(outPath, `${JSON.stringify(artifact, null, 2)}\n`)
-  console.error(`wrote ${outPath}`)
-} else {
-  process.stdout.write(`${JSON.stringify(artifact, null, 2)}\n`)
-}
+await persist(false)
+if (outPath !== undefined) console.error(`wrote ${outPath}`)
+else process.stdout.write(`${JSON.stringify({ definition: definitionPath, cells: rows.length, rows, summary }, null, 2)}\n`)
 
 const blocked = rows.filter((row) => row.status === 'blocked').length
 console.error(`matrix: ${rows.length - blocked} played, ${blocked} blocked`)

retroarch.test.mts

@@ -22,6 +22,7 @@
  */
 import { strict as assert } from 'node:assert'
 import { createHash } from 'node:crypto'
+import { spawnSync } from 'node:child_process'
 import { existsSync, readFileSync } from 'node:fs'
 import { attestRun } from './attestation'
 import { logFrom, observationOf } from './runtime'
@@ -58,18 +59,27 @@ const TRACE_INPUTS = 120
 
 
 function missing(): string | null {
-  // Hard platform guard, ahead of every other check. The RetroArch that
-  // Homebrew installs on macOS is an x86_64 build running under Rosetta, and
-  // it segfaults inside an environment callback during `retro_run`
-  // (KERN_INVALID_ADDRESS, repeatedly, with a crash dialog each time). The
-  // adapter is therefore unproven on darwin and this gate never launches an
-  // emulator there, even when the paths are set. Linux CI is the execution
-  // proof; see docs/adapters.md.
-  if (process.platform === 'darwin') {
-    return 'the RetroArch gate does not run on macOS: the x86_64 build under Rosetta segfaults during retro_run'
-  }
   if (!binary) return 'PLAYPROOF_RETROARCH is unset (path to the RetroArch executable)'
   if (!existsSync(binary)) return `PLAYPROOF_RETROARCH=${binary} does not exist`
+  // The macOS guard tests the ARCHITECTURE, because that is what the failure
+  // was about. An x86_64 RetroArch under Rosetta segfaults inside an
+  // environment callback during `retro_run` (KERN_INVALID_ADDRESS, repeatedly,
+  // with a crash dialog each time), and Homebrew installs exactly that build.
+  //
+  // A guard written as `platform === 'darwin'` states a broader claim than the
+  // evidence supports, and it cost real time: the universal build from
+  // libretro's own stable tree has a native arm64 slice that loads an arm64
+  // gambatte core and the free ROM without incident, which makes this gate
+  // runnable on an Apple Silicon machine and the bug underneath it reproducible
+  // off CI.
+  if (process.platform === 'darwin') {
+    const slices = spawnSync('lipo', ['-archs', binary], { encoding: 'utf8' })
+    const archs = (slices.stdout ?? '').trim().split(/\s+/u)
+    if (slices.status !== 0 || !archs.includes(process.arch === 'arm64' ? 'arm64' : 'x86_64')) {
+      return `the RetroArch gate needs a ${process.arch} slice; ${binary} has [${archs.join(', ') || 'unknown'}],`
+        + ' and an x86_64 build under Rosetta segfaults during retro_run'
+    }
+  }
   if (!core) return 'PLAYPROOF_RETROARCH_CORE is unset (path to a gambatte libretro core)'
   if (!existsSync(core)) return `PLAYPROOF_RETROARCH_CORE=${core} does not exist`
   if (!rom) return 'PLAYPROOF_ROM is unset (path to Libbet and the Magic Floor v0.08)'

retroarch/worker.py

@@ -116,6 +116,10 @@
 # Save and load state are hotkeys with no reply, so both are retried until
 # RetroArch shows the work in its own log or writes the file.
 STATE_ATTEMPTS = 8
+# Restores allowed before a worker refuses to start a run. A restore that lands
+# somewhere else is retried, because the cause is a lost datagram or a retry
+# inside LOAD_STATE rather than anything about the content.
+BOOT_RESTORE_ATTEMPTS = 6
 # A state load makes RetroArch reinitialise its video, input, and audio
 # drivers, and that reinitialisation sometimes ends the process. A reset can
 # therefore replace the emulator, because a reset returns to the pinned boot
@@ -863,6 +867,9 @@ def __init__(self):
         self.gen = 0
         self.frame = 0
         self.boot_blob = None
+        # What the pinned boot instant reads as. Null until the boot state is
+        # pinned, and checked after every restore from then on.
+        self.boot_fingerprint = None
         self.boot_frame = 0
         self.history = []
         self.held = set()
@@ -924,6 +931,24 @@ def _power_on(self):
         self.frame = self.boot_frame + 1
         self.gen += 1
         self._cache = None
+        # Pin what the boot instant LOOKS like, by restoring it once and reading
+        # the channels back. Every later restore must reproduce this exactly.
+        #
+        # WHY. `load_state` retries, and every failed attempt issues a
+        # FRAMEADVANCE that nothing counts, so a load that succeeds on the first
+        # attempt leaves the emulator one frame earlier than one that succeeds
+        # on the second. `reset` then sets `self.frame` unconditionally, so the
+        # worker's counter is right while the emulator is not.
+        #
+        # MEASURED: two same-process replays of one boot state and one input log
+        # agreed byte for byte to emuFrame 811 and then differed on channel
+        # values at IDENTICAL frame numbers, with `ch_c57e_c57f` reading 9000
+        # against 9001 — one tick apart, which is what one uncounted frame does.
+        # The gate failed roughly 40% of the time, and more often under load,
+        # which is when a retry is most likely.
+        self.boot_fingerprint = None
+        self._restore_boot()
+        self.boot_fingerprint = self._read_channels()
 
     def reset(self, seed=None):
         if seed is not None:
@@ -959,12 +984,26 @@ def _restore_boot(self):
         what keeps a replacement out of the evidence.
         """
         self.held = set()
-        try:
-            self._release_all()
-            self.emulator.load_state(self.boot_blob)
-        except RetroArchError:
-            self._relaunch_onto_boot()
-        self.history = []
+        for attempt in range(BOOT_RESTORE_ATTEMPTS):
+            try:
+                self._release_all()
+                self.emulator.load_state(self.boot_blob)
+            except RetroArchError:
+                self._relaunch_onto_boot()
+            self.history = []
+            self._cache = None
+            # Nothing to check against while the fingerprint is being pinned.
+            if self.boot_fingerprint is None:
+                return
+            if self._read_channels() == self.boot_fingerprint:
+                return
+        # Fail rather than play on. A run that starts one frame from where it
+        # believes it starts produces a log that cannot be replayed, and a
+        # divergence discovered later cannot be told from a bad policy.
+        raise RetroArchError(
+            'the boot state did not restore to the pinned instant after %d attempts.'
+            ' The emulator is not where this worker believes it is, so any run from here'
+            ' would not reproduce.' % BOOT_RESTORE_ATTEMPTS)
 
     def identity(self):
         return {

stable-retro.test.mts

@@ -12,6 +12,7 @@
  * spend.
  */
 import { strict as assert } from 'node:assert'
+import { resolve } from 'node:path'
 import { spawnSync } from 'node:child_process'
 import { createHash } from 'node:crypto'
 import { tmpdir } from 'node:os'
@@ -24,7 +25,19 @@ import { RetroRpc } from './adapters/retro-rpc'
 import { bundledReference, makeStableRetro, type RetroState, type StableRetro } from './adapters/stable-retro'
 
 const GAME = 'Airstriker-Genesis'
-const python = process.env.PLAYPROOF_PYTHON ?? 'python3'
+/**
+ * The interpreter, resolved against the CALLER's directory.
+ *
+ * A probe below runs with `cwd: tmpdir()` so it cannot import from the repo by
+ * accident, which means a relative `PLAYPROOF_PYTHON` never resolves and the
+ * gate reports the package missing when the real fault is the path. Measured:
+ * `PLAYPROOF_PYTHON=./.venv/bin/python` produced "stable-retro is not
+ * importable", advising an install of software that was already installed.
+ *
+ * A bare name like `python3` is left alone for PATH lookup.
+ */
+const rawPython = process.env.PLAYPROOF_PYTHON ?? 'python3'
+const python = rawPython.includes('/') ? resolve(rawPython) : rawPython
 
 /**
  * Probe from a temporary directory, never from the repository root: Playproof's