Responsive terminal restore: paced replay, honest gaps, bounded recovery, spill-before-disconnect

.env.example

@@ -114,17 +114,28 @@ PORT=3001
 # Maximum inbound WebSocket frame size in bytes (default: 16777216 = 16MB)
 # WS_MAX_PAYLOAD_BYTES=16777216
 
-# Terminal stream replay ring bytes per terminal (default: 262144 = 256KB)
-# TERMINAL_REPLAY_RING_MAX_BYTES=262144
-
-# Terminal stream output queue bytes per attached client (default: 131072 = 128KB)
-# TERMINAL_CLIENT_QUEUE_MAX_BYTES=131072
+# Terminal stream replay ring bytes per terminal (default: 1048576 = 1MB).
+# This is the per-terminal retained-output ring every attach replays from:
+# negotiated (pacedTerminalReplayV1) clients page it out bounded by
+# credits, and its front bound is what `terminal.attach.ready` reports as
+# `oldestRetainedSeq` (older output is gone; legacy inline attaches just
+# get the retained tail).
+# TERMINAL_REPLAY_RING_MAX_BYTES=1048576
+
+# Terminal stream output queue bytes per attached client (default: 16777216 = 16MB).
+# This is the SPILL bound: past it, oldest output is evicted with a
+# generation-scoped gap instead of disconnecting the client.
+# TERMINAL_CLIENT_QUEUE_MAX_BYTES=16777216
 
 # Terminal stream send batch bytes per flush (default: 65536 = 64KB)
 # TERMINAL_STREAM_BATCH_MAX_BYTES=65536
 
-# Catastrophic WS bufferedAmount threshold for terminal streaming (default: 16777216 = 16MB)
-# TERMINAL_WS_CATASTROPHIC_BUFFERED_BYTES=16777216
+# Catastrophic WS bufferedAmount disconnect threshold (default: 67108864 = 64MB).
+# Must be STRICTLY greater than TERMINAL_CLIENT_QUEUE_MAX_BYTES — the server
+# refuses to start otherwise, so normal pressure spills before any disconnect.
+# The close additionally requires zero successful socket sends for the whole
+# stall window (a slow-but-progressing client is not a dead socket).
+# TERMINAL_WS_CATASTROPHIC_BUFFERED_BYTES=67108864
 
 # Catastrophic stall duration before 4008 close, ms (default: 10000)
 # TERMINAL_WS_CATASTROPHIC_STALL_MS=10000

.github/workflows/rust-tests.yml

@@ -1,6 +1,7 @@
 name: Rust Tests
 
 on:
+  workflow_dispatch:
   pull_request:
 
 permissions:
@@ -35,7 +36,12 @@ jobs:
           # changed is not proof of fixture compatibility. Scoped to the
           # root and workspace members — the independent demo locks are
           # not part of the workspace dependency tree.
-          if git diff --name-only "origin/${{ github.base_ref }}...${{ github.sha }}" \
+          # `github.base_ref` exists only for pull_request events; manual
+          # dispatch (the event-routing escape hatch) compares against the
+          # default branch instead.
+          BASE="${{ github.base_ref }}"
+          [ -n "$BASE" ] || BASE="main"
+          if git diff --name-only "origin/$BASE...${{ github.sha }}" \
             | grep -Eq '\.rs$|(^|/)Cargo\.(toml|lock)$|^rust-toolchain|^\.cargo/|^test/fixtures/|^(pnpm-lock\.yaml|pnpm-workspace\.yaml|package(-lock)?\.json|\.npmrc)$|^(crates|packages)/.*package(-lock)?\.json$|^packages/'; then
             echo "rust=true" >> "$GITHUB_OUTPUT"
           else

AGENTS.md

@@ -264,7 +264,9 @@ stderr note). Provisioning, rotation, and revocation live in
 
 **WebSocket Protocol:** Schema-validated messages using Zod. Handshake flow: client sends `hello` with token → server validates → sends `ready`. Message types include `terminal.create/input/resize/detach/attach` and broadcasts like `sessions.updated`. The `ready` frame carries an optional additive `buildId` (the server's artifact-time-baked git commit, `"unknown"` fallback): the client bakes its own at Vite build time (`__FRESHELL_BUILD_ID__`) and, on a mismatch, reloads exactly once per tab session (sessionStorage sentinel `freshell.server-build-reload` records the last attempted server build id; the same id never reloads twice, a different (corrected) deployment re-arms the guard), self-healing stale-client contract errors; `"unknown"` on either side never triggers or clears the guard (`src/lib/server-build-check.ts`). The once-guard is per server identity: an origin fronted by mixed-build servers could oscillate, and a newer client against an older server costs one futile bounded reload per fresh tab session (both accepted for the single-server self-hosted model).
 
-**PTY Lifecycle:** Each terminal has a unique ID. Server maintains 64KB scrollback buffer. On attach, client receives buffer snapshot then streams new output. On detach, process continues running (background session). Configurable idle timeout (15 mins default).
+**PTY Lifecycle:** Each terminal has a unique ID. The server retains output in a per-terminal replay ring (default 1 MiB, `TERMINAL_REPLAY_RING_MAX_BYTES`; older output is gone). On attach, a non-negotiated (legacy) client receives the retained tail inline as a snapshot then streams new output; on detach, the process continues running (background session). Configurable idle timeout (15 mins default).
+
+**Negotiated Restore (pacedTerminalReplayV1):** A client whose `hello` negotiated `pacedTerminalReplayV1` restores via a bounded paced-replay session instead of the inline snapshot: `terminal.attach.ready` carries the window bounds (plus `oldestRetainedSeq`), a reconnect resumes from the client's coverage cursor (`requestedSinceSeq`/`effectiveSinceSeq`) when its checkpoint is valid, the server pushes one bounded first page, and further pages flow only on `terminal.replay.credit` continuation credits (stale generations and non-negotiated credits are inert). Retention loss mid-restore is reported as an exact `terminal.output.gap` (bounds-carrying), never as a silent truncation or a disconnect. Attach intents pick their own replay shape (viewport hydrate, transport reconnect, hidden-pane `keepalive_delta`; the load-more-history UI flow rides viewport hydrate today), and hidden panes can claim terminals (`terminalLifetimeClaimV1`) so they survive reaping while invisible. Capability negotiation gates all of this: older clients keep the exact legacy wire behavior, and no healthy process is killed or replaced because replay is missing. Per-connection output pressure spills before it disconnects: past 16 MiB of queued output (`TERMINAL_CLIENT_QUEUE_MAX_BYTES`) the oldest frames are evicted with a generation-scoped `terminal.output.gap` (`reason: queue_overflow`; the `ws.terminal_stream.queue_overflow_spill` event is emitted at eviction time, rate-limited per connection), and only a sustained over-64 MiB backlog with zero successful socket sends for the whole stall window closes with 4008 (`ws.terminal_stream.catastrophic_close` carries per-occurrence `sends_in_window` evidence plus the lifetime `total_sends`).
 
 **Claude Session Discovery:** Watches `~/.claude/projects/*/sessions/*.jsonl` for new files. Parses JSONL streams to extract messages, groups by project path.
 

crates/freshell-freshagent/src/opencode_ws.rs

@@ -10087,6 +10087,28 @@ mod tests {
             Some("ses_1"),
             "fixture: the send materialized the durable id (and committed Live{{FreshAgent}})"
         );
+        // The materialize spawn's ownership commit (Live{FreshAgent} on the
+        // durable key) lands asynchronously AFTER the durable id itself;
+        // on a contended 2-core CI runner the test can outrun it (observed
+        // twice on CI: begin_handoff met a not-yet-Live key and the fixture
+        // assert panicked — never on the 96-core dev box). Poll the
+        // registry until the committed precondition is observable instead
+        // of assuming the spawn completed; the bound trips only if the
+        // commit never lands at all.
+        let mut commit_polls = 0u32;
+        loop {
+            let snap = registry.observe("opencode", "ses_1");
+            if matches!(snap.state, freshell_ownership::OwnershipState::Live { .. }) {
+                break;
+            }
+            commit_polls += 1;
+            assert!(
+                commit_polls < 10_000,
+                "fixture: the materialize spawn never committed Live{{FreshAgent}} \
+                 ownership for ses_1: {snap:?}"
+            );
+            tokio::time::sleep(std::time::Duration::from_millis(1)).await;
+        }
         // The refusal precondition: a handoff owns the durable key's
         // transition, so the kill's fenced stop is typed-blocked.
         let freshell_ownership::BeginOutcome::Granted { .. } = registry.begin_handoff(

crates/freshell-protocol/src/client_messages.rs

@@ -32,6 +32,15 @@ pub enum ClientMessage {
     TerminalAttach(TerminalAttach),
     #[serde(rename = "terminal.interest")]
     TerminalInterest(TerminalInterest),
+    /// Responsive-terminal-restore Workstream 1 (paced replay): the
+    /// continuation credit a `pacedTerminalReplayV1` client sends after fully
+    /// consuming an ordered replay page — `consumedSeq` is the last sequence
+    /// it consumed, `attachRequestId` scopes it to one attach generation.
+    /// Additive optional; protocol version stays 10. Ignored by servers that
+    /// predate the capability (accept-and-strip) and by connections whose
+    /// own hello did not negotiate it.
+    #[serde(rename = "terminal.replay.credit")]
+    TerminalReplayCredit(TerminalReplayCredit),
     #[serde(rename = "terminal.autoResumeCancel")]
     TerminalAutoResumeCancel(TerminalAutoResumeCancel),
     #[serde(rename = "terminal.detach")]
@@ -121,7 +130,7 @@ pub enum ClientMessage {
 
 /// The exact `type` discriminants of every client→server message, in the frozen
 /// inventory's order. This is the T0 conformance checklist.
-pub const CLIENT_MESSAGE_TYPES: [&str; 41] = [
+pub const CLIENT_MESSAGE_TYPES: [&str; 42] = [
     "amplifier.activity.list",
     "claude.activity.list",
     "client.diagnostic",
@@ -160,6 +169,7 @@ pub const CLIENT_MESSAGE_TYPES: [&str; 41] = [
     "terminal.input",
     "terminal.interest",
     "terminal.kill",
+    "terminal.replay.credit",
     "terminal.resize",
     "ui.layout.sync",
     "ui.screenshot.result",
@@ -187,6 +197,19 @@ pub struct HelloCapabilities {
     /// advertises the capability back (§4.2). Absent for the frozen client.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub pane_reconcile_v1: Option<bool>,
+    /// Paced terminal restore opt-in (responsive-terminal-restore Workstream
+    /// 1): the client understands bounded, ascending paced replay batches with
+    /// continuation credit. Additive optional — absent on the frozen client
+    /// and stripped-tolerant on older servers (no version bump).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub paced_terminal_replay_v1: Option<bool>,
+    /// Hidden-pane lifetime claims (responsive-terminal-restore Workstream 1):
+    /// the client sends `terminal.interest.claimedTerminalIds` only after the
+    /// `ready` echo advertises the capability back. Additive optional — absent
+    /// on the frozen client and stripped-tolerant on older servers (no version
+    /// bump).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub terminal_lifetime_claim_v1: Option<bool>,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -240,6 +263,14 @@ pub struct TerminalInterest {
     pub revision: u64,
     pub focused_terminal_id: Option<String>,
     pub visible_terminal_ids: Vec<String>,
+    /// Hidden-pane lifetime claims (responsive-terminal-restore Workstream 1,
+    /// negotiated `terminalLifetimeClaimV1` only): terminals this connection
+    /// wants kept alive WITHOUT attaching. `None` carries no claim information
+    /// (the frozen shape); `Some(set)` supersedes the connection's previous
+    /// claim set snapshot-by-snapshot — omitting an id from a later snapshot
+    /// is the explicit withdrawal (release).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub claimed_terminal_ids: Option<Vec<String>>,
 }
 
 // --- client.diagnostic ------------------------------------------------------
@@ -376,6 +407,30 @@ pub struct TerminalAttach {
     pub expected_session_ref: Option<SessionLocator>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub max_replay_bytes: Option<i64>,
+    /// Paced terminal restore (responsive-terminal-restore Workstream 1):
+    /// the negotiated forward-page limit — an optional UPPER BOUND on each
+    /// paced replay page's serialized bytes, honored only on
+    /// pacedTerminalReplayV1 connections and clamped to the server's own
+    /// page-budget cap (`min(requested, server cap)`). Round-2 finding F3:
+    /// the field used to be emitted by the client and silently stripped
+    /// here — it is now part of the honest wire contract. Additive
+    /// optional; a missing, malformed, or non-positive value falls back to
+    /// the server's default exactly like the pre-contract accept-and-strip
+    /// behavior (the lossy deserializer keeps a wrong-typed value from
+    /// failing the whole attach frame). Integer-valued number spellings
+    /// (`2048.0`, `2e3`) carry the same value as their canonical integer
+    /// forms and are accepted and validated the same way (E2R1 finding 3)
+    /// — never silently dropped. E2R1 finding 2 (the honest bound): pages
+    /// are bounded by max(requested, the atomic frame size) — a single
+    /// frame larger than the request forms its own ATOMIC single-frame
+    /// page, bounded by the server's fragment cap (every frame is
+    /// pre-fragmented, so one frame's serialized size never exceeds it).
+    #[serde(
+        default,
+        deserialize_with = "lossy_positive_i64",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub replay_page_bytes: Option<i64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub priority: Option<TerminalAttachPriority>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -404,6 +459,60 @@ pub struct TerminalAttach {
     pub observed_generation: Option<u64>,
 }
 
+/// [`TerminalAttach::replay_page_bytes`]'s lossy deserializer (round-2
+/// finding F3): only a clean positive integer counts as a requested bound;
+/// a missing, malformed (wrong-typed, fractional), or non-positive value
+/// deserializes to `None` — the server's default — instead of failing the
+/// whole attach frame. This preserves the pre-contract accept-and-strip
+/// tolerance for buggy senders exactly.
+///
+/// E2R1 finding 3: integer-VALUED number spellings (`2048.0`, `2e3`)
+/// deserialize through Serde JSON's float storage variant, but they carry
+/// the same VALUE as their canonical integer spellings — JSON has one
+/// number type, and the TS/Zod side (`z.number().int().positive()`) plus
+/// the generated JSON Schema accept that value as an integer. The lossy
+/// deserializer accepts and validates them exactly like the canonical
+/// form; a protocol field a client emits is never silently ignored.
+fn lossy_positive_i64<'de, D>(deserializer: D) -> Result<Option<i64>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let value: Option<serde_json::Value> = Option::deserialize(deserializer)?;
+    Ok(value
+        .as_ref()
+        .and_then(positive_integer_value)
+        .filter(|n| *n > 0))
+}
+
+/// The positive-integer VALUE of one JSON number: Serde JSON's integer
+/// storage directly, or its float storage when the value is integral
+/// (E2R1 finding 3 — `2048.0`/`2e3` parse as floats). Fractional,
+/// non-finite, non-positive, and out-of-i64-range values are `None` (the
+/// malformed/non-positive fallback, never a wrong bound).
+fn positive_integer_value(v: &serde_json::Value) -> Option<i64> {
+    let n = v.as_number()?;
+    if let Some(i) = n.as_i64() {
+        return Some(i);
+    }
+    let f = n.as_f64()?;
+    (f.is_finite() && f.fract() == 0.0 && f > 0.0 && f <= i64::MAX as f64).then_some(f as i64)
+}
+
+/// `terminal.replay.credit` (responsive-terminal-restore Workstream 1): one
+/// continuation credit for a paced replay session, granted after the prior
+/// page was consumed in order. `consumedSeq` must fall within the server's
+/// outstanding-page window `(credited, lastSentPageEnd]`; stale generations
+/// (a superseded `attachRequestId`) and out-of-window values are ignored.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TerminalReplayCredit {
+    pub terminal_id: String,
+    pub stream_id: String,
+    pub attach_request_id: String,
+    /// The last sequence the client fully consumed in order.
+    pub consumed_seq: i64,
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct TerminalDetach {

crates/freshell-protocol/src/lib.rs

@@ -41,7 +41,7 @@ pub use settings::*;
 pub const WS_PROTOCOL_VERSION: u32 = 10;
 
 /// Every `type` discriminant the protocol speaks, both directions, sorted.
-/// (41 client→server + 67 server→client = 108.)
+/// (42 client→server + 67 server→client = 109.)
 pub fn all_message_types() -> Vec<&'static str> {
     let mut types: Vec<&'static str> = client_messages::CLIENT_MESSAGE_TYPES
         .iter()