From 73cb6780f831ae2dd159ce9b4ddd8723ebc76c63 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:02:19 -0700 Subject: [PATCH 01/70] docs(plans): diagnose pane restore backpressure loop and plan responsive restore --- .../2026-09-19-responsive-terminal-restore.md | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 docs/plans/2026-09-19-responsive-terminal-restore.md diff --git a/docs/plans/2026-09-19-responsive-terminal-restore.md b/docs/plans/2026-09-19-responsive-terminal-restore.md new file mode 100644 index 000000000..2d9c7a42b --- /dev/null +++ b/docs/plans/2026-09-19-responsive-terminal-restore.md @@ -0,0 +1,210 @@ +# Reliable, Responsive Pane Restore — Diagnosis and Plan + +Status: proposed (diagnosis complete; no code changes yet) +Date: 2026-09-19 +Incident: live self-hosted server on garageserver (`192.168.3.150:3001`), Electron client on DANDESKTOP +Related: `docs/plans/2026-03-30-paginated-terminal-replay.md`, `docs/plans/2026-07-24-rust-attach-viewport.md` (TERM-07 open items) + +## Summary + +When a client connects, the server resends each terminal pane's entire retained past output in one burst. With the user's layout that is roughly 21 MB across seven terminal panes. The client cannot process it fast enough, the server's "client is too slow" guard fires before its own graceful spill mechanism can run, and the connection is disconnected. The client reconnects and starts over from zero because a restore that never completed leaves no record of how far it got. That loop repeats indefinitely, so panes never finish coming back — including the terminal pane for `opencode --session ses_f5434…`. + +The fix is five workstreams: restore the current screen first and history on demand, make restores resume instead of restarting, spill old output instead of hanging up, provide a real "load earlier history" control, and stop covering panes that already have content (the "Refreshing conversation…" cover). + +## What the user experienced + +- The pane for `opencode --session ses_f5434cdf6ffeZYNTjLVLre77W8` never finished resuming; it appeared hung behind a "Refreshing conversation…" cover. +- Restarting the Electron client did not help: the reconnect/disconnect loop resumed immediately. +- Other panes in the same app showed similar restoring/refreshing symptoms because they share one connection. + +## Diagnosis + +### The failure loop + +1. On connect, the client attaches to every pane in its layout (7 terminal panes plus 7 agent-conversation panes in the incident layout). +2. For each terminal attach, the server resends the terminal's full retained past output ("replay"). Measured ~3 MB per pane. +3. The client asks for only the newest 128 KB per normal terminal pane. The server ignores that request: the field (`maxReplayBytes`) exists in the protocol but no server code reads it. Verified by attaching with a 128 KB request and still receiving ~3 MB. +4. Full-screen terminal apps (opencode) request no limit at all, by design. +5. The connection therefore queues ~21 MB at once. The client's renderer cannot parse that quickly (color-heavy full-screen redraws, plus repeated 6 MB agent-conversation snapshots), so the queue stays above the server's 16 MB threshold for the required 10 seconds. +6. The server disconnects the connection with code 4008 ("catastrophic backpressure"). The client reconnects and repeats from step 1. Progress is discarded each cycle, so there is no convergence. +7. The server's graceful fallback — spill the oldest unsent output — only runs at 32 MB, above the 16 MB disconnect threshold, so it never gets a chance. + +### Evidence + +| Measurement | Value | +| --- | --- | +| Resend for terminal `c1d8efc…` (`ses_f5434…`) | 2,996,228 bytes / 7,491 frames per attach | +| Resend for the other OpenCode terminal (`8ff6bee0…`) | 3,017,089 bytes / 18,805 frames | +| Resend for a Codex terminal (`a6af3bdf…`) | 2,999,964 bytes / 14,487 frames | +| Resend with `maxReplayBytes: 131072` requested | 3,000,000 bytes (cap ignored) | +| Bytes written by the opencode TUI processes over 3 s | ~0 (no live output during the incident) | +| Server queue at disconnect | 21–25 MB vs 16,777,216-byte threshold | +| Disconnect cadence | every 15–20 s, repeating; 28 closures on Sep 18 and continuing Sep 19 | +| Agent-conversation snapshot | 6,015,324 bytes; 6.5 s to fetch for the active pane | + +Supporting source locations: + +- Attach handling: `crates/freshell-ws/src/terminal.rs:6916` (`handle_attach`) → `crates/freshell-terminal/src/registry.rs:1594` (`attach_to_shared`); the full-replay snapshot is taken at `registry.rs:1616-1621`. +- Ignored budget field: `crates/freshell-protocol/src/client_messages.rs:371` (`max_replay_bytes`); no reader anywhere in `crates/freshell-ws` or `crates/freshell-terminal`. +- Client budget request: `src/components/TerminalView.tsx:221` (`TRUNCATED_REPLAY_BYTES = 128 * 1024`) and `:278` (`viewportHydrateReplayOptions`, which returns no limit for opencode). +- Client handling of a budget gap: `src/components/TerminalView.tsx:4335` (`replay_budget_exceeded` → "load more" affordance). +- Thresholds: `crates/freshell-terminal/src/output_queue.rs:22` (queue spill at 32 MB), `crates/freshell-ws/src/backpressure.rs:68-75` (disconnect at 16 MB sustained 10 s), monitor at `crates/freshell-ws/src/terminal.rs:400-410` and close at `:549-565`. +- Graceful gap emission today: `crates/freshell-ws/src/connection_writer.rs:490-498` (queue overflow only; no gap is emitted when the requested position predates the retained history). +- Agent-pane cover: `src/components/fresh-agent/FreshAgentView.tsx:3360` and `:3742` (full-pane overlay while `snapshotDirty`). +- Legacy behavior to restore: `server/terminal-stream/broker.ts:454-481` at commit `a7d36d5f8^` (budget walk newest-to-oldest, `replay_budget_exceeded` gap); the Rust port leaves `maxReplayBytes` unimplemented per the TERM-07 note in `docs/plans/2026-07-24-rust-attach-viewport.md`. + +### Why it cannot recover + +Three design choices combine into a permanent loop: + +1. Every connect resends everything (no budget honored). +2. Every failure forgets progress (a partial restore leaves no usable position, so the retry is a full replay). +3. Every slow client gets disconnected (the kill fires before the spill). + +Fixing only the ignored budget shrinks the burst but leaves (2) and (3) able to reproduce the loop under other load. + +### Not the cause + +- The OpenCode session is healthy: idle, last turn completed 17:21 UTC, no error state. +- The terminal programs are not flooding: their own writes were ~0 bytes during the incident. The traffic is entirely resent history. +- The server is up and serving; this is not a crash or restart problem. +- The client/server build mismatch is real but handled correctly (the one-shot reload guard suppresses further reloads). It is unrelated. + +### How to reproduce + +From a machine that can reach the server, open a WebSocket to `/ws`, complete the `hello` handshake (protocol version 10, token), and send `terminal.attach` for a terminal with a full scrollback, with and without `maxReplayBytes`. Count the payload bytes of the `terminal.output.batch` messages for five seconds: + +- Without the field: ~3 MB arrives in under a second, then silence. +- With `maxReplayBytes: 131072`: the same ~3 MB arrives — proof the server ignores the request. + +To reproduce the user-visible loop, open the app with several full-scrollback terminal panes and watch the server log for `ws.terminal_stream.catastrophic_close` followed by `ws.connection.closed reason=catastrophic_backpressure code=4008`, then a fresh `ws.connection.established` — repeating. + +## Goals + +What the person at the keyboard should get: + +1. Opening Freshell with many busy panes: the pane they are looking at is alive in about a second and accepts typing immediately. +2. Panes that are not visible cost almost nothing. +3. Scrolling up loads older output in small pieces without stalling anything; there is an honest "load earlier" control when more exists. +4. A dropped connection keeps what was already shown; on reconnect only the missing part is fetched, and the restore always finishes. +5. No pane is covered by a full-screen "refreshing" curtain while it has content to show. + +## Plan + +### Workstream 1 — Send the current screen first, history later (server; client) + +The client already asks for a small slice; make the server honor it, for every pane type. + +Implementation notes: + +- Parse `max_replay_bytes` in `handle_attach` and thread it through `attach_with_geometry` / `attach_to_shared` (`crates/freshell-ws/src/terminal.rs:6916`, `crates/freshell-terminal/src/registry.rs:1558`). +- Keep the newest frames within the budget, measured the way the wire measures them (serialized payload bytes, per the legacy broker at `broker.ts:454-481`), walking newest-to-oldest and stopping at the first frame that does not fit. +- Start the kept slice at a safe boundary (do not begin mid-escape-sequence), and let the existing modes preamble (surface reset) precede the replay as it does today. +- Report the dropped span with the existing gap message and `replay_budget_exceeded`; the client already turns that into a "load more history" affordance (`TerminalView.tsx:4335`). Set `replayFromSeq` / `replayToSeq` to the kept span. +- Add a per-connection total startup budget as a backstop so a workspace with many panes stays bounded regardless of tab count. +- Include opencode: have the client request a budget for opencode panes too, or apply a server-side default cap. The newest output of these full-screen apps contains complete repaints of the current screen, so a small slice reconstructs it. If a pane has been quiet (no repaint in the newest slice), trigger one repaint (a resize nudge) before capturing the replay. Verify with a screen-reconstruction test on the real TUIs before turning off the exemption. +- Client-side: hidden panes should request little or nothing until revealed; on reveal, top up from the pane's position. + +Tests: + +- Registry unit tests: newest-first budget walk; exact boundary behavior; gap range for the dropped span; empty/small budgets. +- WS integration: attach with a budget returns payload ≤ budget plus the gap frame; `replayFromSeq`/`replayToSeq` match the kept span. +- A screen-reconstruction test for opencode (newest slice + preamble yields the same visible screen as a full replay). + +### Workstream 2 — Resume instead of restarting (client + server) + +Implementation notes: + +- The client currently resets its output position to zero on every viewport hydrate (`TerminalView.tsx:3098`), so a hydrate interrupted by a disconnect restarts from zero. Record the applied position per pane even when a hydrate is incomplete, and on retry ask only for what comes after it (the server already honors `since_seq` at `registry.rs:1616-1621`). +- Keep a full replay only when the terminal surface itself was recreated (page load or user reset). +- The server must tell the client when the requested position is older than the retained window: emit the existing gap message for that span (today only queue overflow emits gaps, `connection_writer.rs:490-498`). Without this, a resumed attach could silently miss output. +- Bound retries and surface an explicit "couldn't finish loading — retry" state instead of looping silently. + +Tests: + +- Client unit: an interrupted hydrate retries with the applied position, not zero; a recreated surface still full-replays. +- Integration: drop the connection mid-replay, reconnect, assert the second attach requests only the remainder and the pane converges to the same content. +- Integration: request a position older than the retained window and assert a gap is reported. + +### Workstream 3 — Spill old output instead of hanging up (server) + +Implementation notes: + +- Today the queue spills at 32 MB but the connection is killed at 16 MB, so the graceful path never runs (`output_queue.rs:22`, `backpressure.rs:68-75`). Make the spill threshold lower than the disconnect threshold, or exclude resent history from the disconnect measure. +- Disconnect only when the connection makes no drain progress at all for a long window — slow processing is not a dead socket. +- Keep the existing spill behavior (drop oldest unsent output, emit a gap) as the primary response to overload. + +Tests: + +- A deliberately slow consumer stays connected and receives gaps instead of a close. +- A genuinely dead consumer is still closed. +- Queue memory stays bounded under a replay burst larger than both thresholds. + +### Workstream 4 — Load earlier history on demand (client + server) + +Implementation notes: + +- The "load more history" affordance and gap state already exist; today the only fill path is a full re-attach. Add a bounded "fetch older output before position X" request (design decision: a new client message versus extending the attach path). +- Fetch in fixed-size chunks on a low-priority lane; preserve scroll position while prepending; show an honest boundary line while earlier output is not loaded. +- Only the requesting pane is affected; other panes never fetch. + +Tests: + +- E2E: scroll to the top of a long terminal, load a chunk, repeat — history walks backwards without stalls, and the boundary is visible until fully loaded. + +### Workstream 5 — Stop covering panes that have content (client + server) + +Implementation notes: + +- Replace the full-pane overlay in `FreshAgentView.tsx:3360`/`:3742` with an in-place refresh: render the last known conversation immediately and show a small status line while updating. +- Paginate the conversation snapshot endpoint (`/api/fresh-agent/threads/...`, currently 6 MB): recent turns first, older turns on demand, with a cursor. +- Hidden panes do not poll or fetch at all; on reveal, fetch the recent slice only. + +Tests: + +- Client: a revealed pane shows its last known content instantly while refreshing (no full-pane cover). +- Server: snapshot payloads are bounded and support fetching older turns. +- Client: hidden panes issue no requests. + +## Sequencing + +1. Workstreams 1 and 2 together — the correctness floor: bounded restores that always finish. +2. Workstream 3 — small server change that removes the last way to get stuck. +3. Workstream 4 — the history-loading experience. +4. Workstream 5 — the agent-pane experience. + +## Acceptance criteria + +End-user: + +- With the incident layout (7 terminals + 7 agent panes), the active pane is interactive in about a second; total startup traffic is a few hundred KB per pane; no disconnects; no multi-second freezes. +- Cutting the network mid-restore: the UI keeps showing what it had, and the restore completes after reconnect without resending everything. +- Scrolling to the top of a long terminal loads chunks without stalling and shows an honest boundary. +- No pane is ever covered by a full-screen refreshing curtain while it has content. + +Automated: + +- Per-pane attach payloads stay within the requested budget; a budget gap is reported. +- A slow consumer receives gaps, never a disconnect; a dead consumer is still disconnected. +- A retry after a mid-restore drop requests only the remainder. +- Hidden panes neither attach nor poll. + +## Risks and open questions + +- Alt-screen reconstruction: a small slice of a full-screen app must reliably rebuild the visible screen. Validate on opencode, Codex, and Claude TUIs; keep the repaint nudge as the fallback if a quiet pane's newest slice does not cover the screen. +- Budget defaults: 128 KB for visible panes; decide the hidden-pane and per-connection totals. +- History fetch transport: new WebSocket message versus an HTTP endpoint; pick during implementation. +- Retry ceiling: how many attempts before showing the explicit failure state. +- Deployment: server-side changes require a restart of the live 3.150 server, which needs the user's explicit approval. +- Base health: the coordinator's latest full-suite run is red for reasons unrelated to this document; re-check the base before implementing. + +## Appendix — incident log excerpts + +``` +WARN ws.terminal_stream.catastrophic_close pending_bytes=21754500 threshold=16777216 +INFO ws.connection.closed reason=catastrophic_backpressure code=4008 +INFO ws.connection.established +... repeating every 15-20 s +``` + +Client console during the loop: `xterm.js: Parsing error` entries while re-parsing replay data on each reconnect; the app remained connected but could not finish restoring. From 921c736adbfe047df66cb2d958e66ae0d962676d Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:02:19 -0700 Subject: [PATCH 02/70] docs(plans): define safe responsive terminal restore contracts --- .../2026-09-19-responsive-terminal-restore.md | 213 ++++++++++++------ 1 file changed, 138 insertions(+), 75 deletions(-) diff --git a/docs/plans/2026-09-19-responsive-terminal-restore.md b/docs/plans/2026-09-19-responsive-terminal-restore.md index 2d9c7a42b..d2a00467d 100644 --- a/docs/plans/2026-09-19-responsive-terminal-restore.md +++ b/docs/plans/2026-09-19-responsive-terminal-restore.md @@ -1,15 +1,17 @@ # Reliable, Responsive Pane Restore — Diagnosis and Plan -Status: proposed (diagnosis complete; no code changes yet) +Status: revised proposal after source review; screen reconstruction and history representation require design validation before implementation (no code changes yet) Date: 2026-09-19 Incident: live self-hosted server on garageserver (`192.168.3.150:3001`), Electron client on DANDESKTOP Related: `docs/plans/2026-03-30-paginated-terminal-replay.md`, `docs/plans/2026-07-24-rust-attach-viewport.md` (TERM-07 open items) ## Summary -When a client connects, the server resends each terminal pane's entire retained past output in one burst. With the user's layout that is roughly 21 MB across seven terminal panes. The client cannot process it fast enough, the server's "client is too slow" guard fires before its own graceful spill mechanism can run, and the connection is disconnected. The client reconnects and starts over from zero because a restore that never completed leaves no record of how far it got. That loop repeats indefinitely, so panes never finish coming back — including the terminal pane for `opencode --session ses_f5434…`. +When a client attaches to a terminal, the server sends its entire retained output newer than the requested position and ignores the requested replay budget. In the incident, initial and repeated full hydrations amounted to roughly 21 MB across seven terminals. The server's sustained-backlog disconnect threshold is lower than its output-spill threshold, creating a reconnect loop. The client already records parser-applied progress, but an incomplete fresh-surface hydrate and other safety checks can force the next attempt back to zero. -The fix is five workstreams: restore the current screen first and history on demand, make restores resume instead of restarting, spill old output instead of hanging up, provide a real "load earlier history" control, and stop covering panes that already have content (the "Refreshing conversation…" cover). +The fix must bound delivery without treating arbitrary terminal output as a screen snapshot. First establish a correct reconstruction and resume contract, then implement bounded delivery, resumable hydration, and backpressure handling together. Terminal history browsing and conversation pagination are separate follow-on work. Removing the conversation refresh cover also requires fixing the same-revision refresh retry condition. + +This revision incorporates source review of the existing client checkpoints, terminal queues, OpenCode recovery, and conversation refresh paths. Incident measurements below are retained from the original investigation; the review did not repeat live probes or establish how much of the network backlog was caused specifically by renderer work. ## What the user experienced @@ -17,16 +19,18 @@ The fix is five workstreams: restore the current screen first and history on dem - Restarting the Electron client did not help: the reconnect/disconnect loop resumed immediately. - Other panes in the same app showed similar restoring/refreshing symptoms because they share one connection. +The "Refreshing conversation…" cover is implemented by `FreshAgentView`, whereas terminal replay is handled by `TerminalView`. Treat these as separate recovery paths observed in the same incident; record pane IDs and content types in the reproduction rather than attributing the conversation overlay directly to terminal replay. + ## Diagnosis ### The failure loop -1. On connect, the client attaches to every pane in its layout (7 terminal panes plus 7 agent-conversation panes in the incident layout). -2. For each terminal attach, the server resends the terminal's full retained past output ("replay"). Measured ~3 MB per pane. +1. Initial connection and subsequent hydration schedule terminal attachments across the layout (7 terminals plus 7 agent-conversation panes in the incident). Existing background hydration/rebind queues already stage some work; this is not one unconditional synchronous attach of every pane. +2. For each zero-position terminal hydrate, the server resends the terminal's full retained past output ("replay"). Measured ~3 MB per pane. 3. The client asks for only the newest 128 KB per normal terminal pane. The server ignores that request: the field (`maxReplayBytes`) exists in the protocol but no server code reads it. Verified by attaching with a 128 KB request and still receiving ~3 MB. 4. Full-screen terminal apps (opencode) request no limit at all, by design. -5. The connection therefore queues ~21 MB at once. The client's renderer cannot parse that quickly (color-heavy full-screen redraws, plus repeated 6 MB agent-conversation snapshots), so the queue stays above the server's 16 MB threshold for the required 10 seconds. -6. The server disconnects the connection with code 4008 ("catastrophic backpressure"). The client reconnects and repeats from step 1. Progress is discarded each cycle, so there is no convergence. +5. The observed server output backlog is ~21–25 MB and remains above 16 MB for the required 10 seconds. The writer measures queued plus in-flight terminal output, not xterm's parsing progress. Color-heavy replay and large HTTP conversation snapshots are additional client work, but HTTP snapshot bytes are not part of that terminal queue measurement. +6. The server disconnects with code 4008 ("catastrophic backpressure"). Reconnect can force another fresh-surface hydrate, which resets the usable replay position despite some output having reached the parser; the observed attempts do not converge. 7. The server's graceful fallback — spill the oldest unsent output — only runs at 32 MB, above the 16 MB disconnect threshold, so it never gets a chance. ### Evidence @@ -51,15 +55,15 @@ Supporting source locations: - Thresholds: `crates/freshell-terminal/src/output_queue.rs:22` (queue spill at 32 MB), `crates/freshell-ws/src/backpressure.rs:68-75` (disconnect at 16 MB sustained 10 s), monitor at `crates/freshell-ws/src/terminal.rs:400-410` and close at `:549-565`. - Graceful gap emission today: `crates/freshell-ws/src/connection_writer.rs:490-498` (queue overflow only; no gap is emitted when the requested position predates the retained history). - Agent-pane cover: `src/components/fresh-agent/FreshAgentView.tsx:3360` and `:3742` (full-pane overlay while `snapshotDirty`). -- Legacy behavior to restore: `server/terminal-stream/broker.ts:454-481` at commit `a7d36d5f8^` (budget walk newest-to-oldest, `replay_budget_exceeded` gap); the Rust port leaves `maxReplayBytes` unimplemented per the TERM-07 note in `docs/plans/2026-07-24-rust-attach-viewport.md`. +- Legacy budget behavior: `server/terminal-stream/broker.ts:454-481` at commit `a7d36d5f8^` (budget walk newest-to-oldest, `replay_budget_exceeded` gap); the Rust port leaves `maxReplayBytes` unimplemented per the TERM-07 note in `docs/plans/2026-07-24-rust-attach-viewport.md`. Restoring the tail selection alone is insufficient for screen correctness and resumable progress. ### Why it cannot recover Three design choices combine into a permanent loop: -1. Every connect resends everything (no budget honored). -2. Every failure forgets progress (a partial restore leaves no usable position, so the retry is a full replay). -3. Every slow client gets disconnected (the kill fires before the spill). +1. A full hydrate sends the entire retained ring because its budget is ignored; an already-valid delta attach does honor `since_seq`. +2. An incomplete fresh-surface hydrate can force another full hydrate. Furthermore, reporting a skipped prefix through the existing gap handler prevents the parser-applied checkpoint from advancing past that prefix. +3. Sustained terminal backlog can trigger disconnect before spilling begins, even if socket sends are still making progress. Fixing only the ignored budget shrinks the burst but leaves (2) and (3) able to reproduce the loop under other load. @@ -72,7 +76,7 @@ Fixing only the ignored budget shrinks the burst but leaves (2) and (3) able to ### How to reproduce -From a machine that can reach the server, open a WebSocket to `/ws`, complete the `hello` handshake (protocol version 10, token), and send `terminal.attach` for a terminal with a full scrollback, with and without `maxReplayBytes`. Count the payload bytes of the `terminal.output.batch` messages for five seconds: +For future reproduction, use a disposable test instance populated with equivalent scrollback, not live user terminals. Open a WebSocket to `/ws`, complete the `hello` handshake (protocol version 10, token), and send `terminal.attach` for a terminal with a full scrollback, with and without `maxReplayBytes`. Count the payload bytes of the `terminal.output.batch` messages for five seconds. The original incident probe observed: - Without the field: ~3 MB arrives in under a second, then silence. - With `maxReplayBytes: 131072`: the same ~3 MB arrives — proof the server ignores the request. @@ -83,120 +87,179 @@ To reproduce the user-visible loop, open the app with several full-scrollback te What the person at the keyboard should get: -1. Opening Freshell with many busy panes: the pane they are looking at is alive in about a second and accepts typing immediately. -2. Panes that are not visible cost almost nothing. -3. Scrolling up loads older output in small pieces without stalling anything; there is an honest "load earlier" control when more exists. -4. A dropped connection keeps what was already shown; on reconnect only the missing part is fetched, and the restore always finishes. +1. Opening Freshell with many panes: target an interactive, correct active screen in about a second on the incident hardware. Measure this; a bounded network payload alone does not establish responsiveness. +2. Hidden panes do not hydrate terminal screens or fetch conversation bodies until revealed. Cheap lifecycle, ownership, and status subscriptions remain available. +3. Earlier retained history is available in bounded pages without disturbing the live terminal; distinguish unloaded history from history that has expired. +4. A dropped connection keeps what was already shown. With compatible terminal state and retained missing output, reconnect resumes from applied progress. If reconstruction is impossible, preserve the view and show an explicit recovery state instead of silently looping or restarting a healthy process. 5. No pane is covered by a full-screen "refreshing" curtain while it has content to show. ## Plan -### Workstream 1 — Send the current screen first, history later (server; client) +### Shared restore contract — implement before enabling truncation -The client already asks for a small slice; make the server honor it, for every pane type. +Terminal output is a stateful instruction stream. An escape-sequence boundary is not necessarily a point from which a blank terminal can reconstruct cells, cursor position, text attributes, scroll regions, or alternate-screen state. Existing `terminal.modes.sync` restores mode flags only. The server currently has no canonical screen snapshot that proves a retained suffix is independently renderable. -Implementation notes: +Use these distinctions throughout server responses, client state, history controls, and tests: + +| Condition | Meaning | Required client behavior | +| --- | --- | --- | +| Intentionally unloaded older history (`replay_budget_exceeded`) | Older bytes remain fetchable, and the current screen has an independently validated baseline | Show a history boundary; do not invalidate that baseline merely because older history is unloaded | +| Retention loss (`replay_window_exceeded`) | Requested bytes have expired | Mark the unavailable span honestly; resume only from a replacement valid baseline, otherwise retain the visible surface and show recovery state | +| Delivery loss (`queue_overflow`) | This connection missed sequenced output | Repair from retained output or a valid screen snapshot; never silently advance the applied cursor across the gap | + +The existing `onOutputGap` records every span as a lost range and `markParserAppliedSeq` cannot cross it (`src/lib/terminal-attach-seq-state.ts`). Thus emitting a budget gap for `1..N` currently leaves a fresh surface's applied cursor at zero even after its retained tail is rendered. Add explicit baseline coverage and history availability to the state model; do not fix this by treating missing parser input as applied. A raw tail without a validated baseline remains incomplete, even if it looks plausible. + +Scope all restore state to terminal ID, stream ID, server identity/boot, surface generation, attach generation, and compatible geometry. Responses must identify requested/effective sequence bounds, current head, oldest retained sequence, and any lost interval. Define additive capability negotiation for new reconstruction, continuation, and gap semantics before enabling them. Older clients must not receive newly introduced retention gaps that enter their automatic OpenCode replacement path; retain compatible behavior until negotiation selects the new contract. + +Evidence for these constraints: `TerminalView.tsx` (`completeParserAppliedFrame`, `attachTerminal`, `beginOpenCodeReplacementAfterExit`); `src/lib/terminal-surface-checkpoint.ts`; `test/unit/client/lib/terminal-attach-seq-state.test.ts` (lost-prefix checkpoint remains zero); `crates/freshell-terminal/src/registry.rs` (`apply_attach_geometry`, `attach_to_shared`, reader ingestion); `crates/freshell-terminal/src/mode_tracker.rs`; and the append-only limitation recorded in `docs/plans/2026-03-30-paginated-terminal-replay.md`. + +### Workstream 1 — Bound restore delivery and establish a correct screen (server + client) + +Deliver in two increments. Paced replay fixes the unbounded burst; a screen snapshot is needed to make fresh TUI restoration independent of history size. The first increment must not be presented as meeting the final one-second screen-first goal. + +1. **Paced forward restoration.** Negotiate a continuation/credit capability (proposed name: `pacedTerminalReplayV1`) that sends bounded, ascending sequence batches from a compatible applied cursor or a proven fresh-surface baseline. Parser acknowledgements carry terminal, stream, attach generation, and the last fully applied sequence; stale acknowledgements or values beyond the sent window cannot grant credit. Grant more replay credit only after xterm applies the prior batch. Keep a fixed initial catch-up target so ongoing live output cannot move the completion condition indefinitely; subsequently deliver live output in sequence order. Do not let live frames for the same terminal overtake unapplied replay. Input and other panes' traffic remain schedulable. +2. **Validated current-screen restoration.** Prototype a server-maintained terminal emulator/checkpoint, updated in PTY output order, that can reconstruct both normal and alternate buffers plus cursor, attributes, modes, and geometry. A snapshot carries its stream/geometry identity and covered sequence; snapshot capture and subscription installation must establish a gap-free snapshot→delta handoff. Compare its output against the real client parser before selecting the implementation. Do not add an emulator dependency solely on the assumption that it is xterm-compatible. -- Parse `max_replay_bytes` in `handle_attach` and thread it through `attach_with_geometry` / `attach_to_shared` (`crates/freshell-ws/src/terminal.rs:6916`, `crates/freshell-terminal/src/registry.rs:1558`). -- Keep the newest frames within the budget, measured the way the wire measures them (serialized payload bytes, per the legacy broker at `broker.ts:454-481`), walking newest-to-oldest and stopping at the first frame that does not fit. -- Start the kept slice at a safe boundary (do not begin mid-escape-sequence), and let the existing modes preamble (surface reset) precede the replay as it does today. -- Report the dropped span with the existing gap message and `replay_budget_exceeded`; the client already turns that into a "load more history" affordance (`TerminalView.tsx:4335`). Set `replayFromSeq` / `replayToSeq` to the kept span. -- Add a per-connection total startup budget as a backstop so a workspace with many panes stays bounded regardless of tab count. -- Include opencode: have the client request a budget for opencode panes too, or apply a server-side default cap. The newest output of these full-screen apps contains complete repaints of the current screen, so a small slice reconstructs it. If a pane has been quiet (no repaint in the newest slice), trigger one repaint (a resize nudge) before capturing the replay. Verify with a screen-reconstruction test on the real TUIs before turning off the exemption. -- Client-side: hidden panes should request little or nothing until revealed; on reveal, top up from the pane's position. +The paced path can reconstruct exactly only if its starting state and required bytes are available. The retained ring may already have lost essential state, so replaying its entire remaining tail is not a proof of correctness either. In that case use a validated snapshot when available, or show an explicit incomplete-screen state with retry; do not restart the program automatically. A newly introduced server emulator cannot retroactively reconstruct missing history for an already-running stream: mark its initial baseline unknown until a verified reset/repaint boundary, or use the explicit unavailable state. + +Implementation requirements: + +- Thread `max_replay_bytes` through both geometry-authorized and geometry-skipped attach paths in `handle_attach` and the registry. Preserve the legacy field's serialized application-JSON tail-budget meaning (confirmed in `broker.ts:454-481` at `a7d36d5f8^`); introduce negotiated forward-page limits rather than silently redefining it as a continuation protocol. Apply newest-tail selection only where the snapshot/baseline contract proves it correct. +- Specify terminal-data UTF-8 bytes separately from serialized message bytes. The existing replay-frame byte count is not the full JSON wire size. Enforce a serialized delivery budget including escaping, batch metadata, and envelope; account for ready/gap/mode controls separately under bounded control limits. Cover multibyte text, JSON escaping, empty replay, zero/undersized budgets, and a frame larger than the budget. Return a bounded explicit result rather than sending an oversized frame or an endless zero-progress page; any frame fragmentation needs sequence plus offset semantics and reassembly tests. +- Bound both per-pane unacknowledged replay and total per-connection admitted bytes, including snapshot/replay copies and the frame being sent. Select pages before cloning payloads; do not first allocate one full replay copy per pane. Cancellation, superseded attaches, and disconnect release credits and temporary state. +- Do not pin an unbounded replay backlog for a slow client. If retention overtakes a continuation cursor, report the exact lost interval and enter bounded baseline recovery. A finite retention window cannot guarantee convergence against indefinitely faster output production. +- Remove the proposed resize nudge. Attach resize and replay capture currently share the terminal lock needed by the output reader; an immediate capture cannot include the asynchronous repaint, and waiting under that lock prevents ingestion. Same-size resize is a no-op, and another subscriber may own geometry. Any future repaint protocol must await a verifiable result outside that lock, respect geometry ownership, and fall back on timeout; it is not a prerequisite for the paced path. +- Hidden panes do not request screen hydration until revealed. Preserve cheap lifecycle/status subscriptions separately from output delivery and terminal lifetime. Do not use ordinary `terminal.detach` as a visibility toggle without checking its release/reap semantics. Remove hidden hydration-queue grants rather than sending a nominal zero budget that the current truthy serializer omits. On reveal, use a valid retained surface checkpoint or request a fresh baseline. Tests: -- Registry unit tests: newest-first budget walk; exact boundary behavior; gap range for the dropped span; empty/small budgets. -- WS integration: attach with a budget returns payload ≤ budget plus the gap frame; `replayFromSeq`/`replayToSeq` match the kept span. -- A screen-reconstruction test for opencode (newest slice + preamble yields the same visible screen as a full replay). +- Registry/WS: byte budgets, continuation bounds, fixed catch-up target, snapshot→live ordering under concurrent output, expired continuation, cancellation, and bounded aggregate memory with many panes. +- Client/WS: withhold parser callbacks to prove replay credit does not advance on receipt; release callbacks and prove forward progress without duplicate output or same-terminal reordering. +- Reconstruction: compare visible cells, cursor, attributes, modes, and subsequent input behavior against an uninterrupted xterm reference using shell, OpenCode, Codex, and Claude recordings, plus real TUI smoke tests. Include quiet/no-repaint output, split control sequences, alternate-buffer transitions, different geometry, second viewers, and screen representations larger than the nominal budget. +- Compatibility: new/old client-server pairs select supported behavior; lack of capability never causes an automatic kill or claims a raw suffix is a complete screen. ### Workstream 2 — Resume instead of restarting (client + server) Implementation notes: -- The client currently resets its output position to zero on every viewport hydrate (`TerminalView.tsx:3098`), so a hydrate interrupted by a disconnect restarts from zero. Record the applied position per pane even when a hydrate is incomplete, and on retry ask only for what comes after it (the server already honors `since_seq` at `registry.rs:1616-1621`). -- Keep a full replay only when the terminal surface itself was recreated (page load or user reset). -- The server must tell the client when the requested position is older than the retained window: emit the existing gap message for that span (today only queue overflow emits gaps, `connection_writer.rs:490-498`). Without this, a resumed attach could silently miss output. -- Bound retries and surface an explicit "couldn't finish loading — retry" state instead of looping silently. +- Reuse the existing parser-applied checkpoint machinery. `completeParserAppliedFrame` saves progress after xterm's write callback; receipt/highest-observed sequence is not sufficient. The defect is that `surfaceFreshRef` can override an otherwise usable partial checkpoint and force zero replay, not that progress recording is entirely absent. +- Represent incomplete hydration separately from a newly created blank surface. A compatible partial hydrate can resume on the same mounted xterm without clearing it, replaying a mode preamble, or claiming `surfaceReset` again. Applying a complete validated snapshot establishes its covered-sequence baseline only after the parser applies it; incomplete snapshot installation must not be mistaken for resumable terminal deltas. +- Retain `canUseCheckpointForDeltaReplay` checks for terminal/stream/server identity, surface generation, geometry and authority, scrollback, parser readiness, and xterm version. Recreated surfaces, changed streams, or incompatible geometry need a new valid baseline, even if the old sequence is nonzero. Scope checkpoints to the actual surface so sibling panes showing the same terminal cannot borrow each other's rendered progress. +- When writes are in flight, freeze generation transitions and wait boundedly for the owned queue to drain before deciding whether its applied checkpoint remains usable. Do not clear a surface while an earlier write can still mutate it. If current quarantine rules have already invalidated that generation, preserve quarantine and rebuild from a valid baseline rather than accepting stale callbacks. +- Implement reason-aware gaps from the shared contract. In particular, remove the `replay_window_exceeded` → `beginOpenCodeReplacementAfterExit` automatic kill path before enabling new server retention-gap notifications. Restore gaps must not emit `terminal.kill`, change terminal identity, or spawn a replacement for a healthy process. Any explicit restart action remains separate user intent. +- Bound automatic recovery by attempts and lack of applied progress. Reset recovery accounting on genuine parser progress or explicit retry, not merely on receiving `attach.ready` or another reconnect. Preserve visible content and show an accessible failure/retry state when the limit is reached. Tests: -- Client unit: an interrupted hydrate retries with the applied position, not zero; a recreated surface still full-replays. -- Integration: drop the connection mid-replay, reconnect, assert the second attach requests only the remainder and the pane converges to the same content. -- Integration: request a position older than the retained window and assert a gap is reported. +- Interrupt a hydrate after some xterm callbacks: same surface requests only the remainder, without clear/reset/preamble, and converges to the uninterrupted reference. +- Interrupt before a callback, delay old callbacks across recovery, and independently change each checkpoint identity/geometry field: no false progress, duplicate writes, or stale checkpoint acceptance. Page reload and sibling surfaces cannot reuse a cursor without their corresponding state. +- Start with a validated bounded screen and unloaded history, apply live output, reconnect: the applied cursor advances while the history boundary remains. A true delivery gap still blocks unsafe advancement. +- A healthy idle OpenCode terminal with expired retained output reports incomplete history/screen state as appropriate; terminal ID and process remain unchanged and subsequent live output still arrives. Repeat for a queue gap; separately verify the compatible path for an older unnegotiated client. +- Recovery with no progress reaches a visible retry state and does not enter an automatic kill/recreate or reconnect loop. ### Workstream 3 — Spill old output instead of hanging up (server) Implementation notes: -- Today the queue spills at 32 MB but the connection is killed at 16 MB, so the graceful path never runs (`output_queue.rs:22`, `backpressure.rs:68-75`). Make the spill threshold lower than the disconnect threshold, or exclude resent history from the disconnect measure. -- Disconnect only when the connection makes no drain progress at all for a long window — slow processing is not a dead socket. -- Keep the existing spill behavior (drop oldest unsent output, emit a gap) as the primary response to overload. +- Set consistent byte limits so normal output pressure reaches bounded admission/spill before any pressure-related disconnect. Include the leased in-flight frame; `DeliveryQueue::outstanding_bytes` already charges reserved bytes. Keep independent hard bounds on metadata/control mailboxes. Validate environment overrides so a configuration cannot silently restore the 32 MB spill / 16 MB disconnect mismatch. +- Replace the sustained-byte-count-only disconnect decision for healthy draining clients. Preserve the existing independent socket-send timeout and keepalive failure handling; slow consumption alone is not a dead socket. If tracking drain progress, count successful sends, not falling queue size: eviction and superseded attachments also reduce queue bytes. Parser credit from workstream 1 governs restore production independently of socket liveness. +- Keep ordered, generation-scoped overflow gaps and non-evictable sequenced controls. A spill bounds memory but does not repair a terminal screen: route the loss through workstream 2's repair contract. Do not exempt replay from memory accounting or silently drop control frames. +- Reuse existing focused/visible/background byte-fair scheduling. Keep input, completion/status, and other panes responsive while a terminal restores; avoid a second competing priority scheduler. Tests: -- A deliberately slow consumer stays connected and receives gaps instead of a close. -- A genuinely dead consumer is still closed. -- Queue memory stays bounded under a replay burst larger than both thresholds. +- A slow consumer that continues making sends stays connected under sustained replay pressure; ordinary replay is paced and overflow produces an exact gap when necessary. +- A genuinely blocked send and a missing keepalive response still close the connection; eviction alone cannot masquerade as send progress. +- Queue + in-flight + restore-state memory and control/metadata counts stay bounded under a burst larger than all limits. Include an oversized indivisible frame and conflicting environment overrides. +- A gap followed by repair yields the correct screen; focused-pane input and sequenced exit/status delivery survive other panes' backlog. ### Workstream 4 — Load earlier history on demand (client + server) -Implementation notes: +Implementation direction: -- The "load more history" affordance and gap state already exist; today the only fill path is a full re-attach. Add a bounded "fetch older output before position X" request (design decision: a new client message versus extending the attach path). -- Fetch in fixed-size chunks on a low-priority lane; preserve scroll position while prepending; show an honest boundary line while earlier output is not loaded. -- Only the requesting pane is affected; other panes never fetch. +- Use a separate read-only history surface within the pane, with a clear return-to-live control. xterm has no supported prepend operation; `handleLoadMoreHistory` currently clears and reattaches from zero. Do not feed backwards pages into the live terminal or its forward-only sequence/cursor state. +- Before implementation, select a historical content representation: independently renderable normal-buffer rows from the validated server emulator/checkpoint machinery, or pages reconstructed from a retained parser checkpoint plus subsequent output. Raw ANSI fragments alone are not independently renderable history. Bound checkpoint/row retention alongside replay retention; no unlimited archival store is included in this change. Do not manufacture older content that was never retained. +- Add a read-only, byte-bounded earlier-history endpoint with a terminal/stream-scoped cursor, exclusive before-position, oldest available position, and explicit exhausted/expired outcomes. Prefer the existing authenticated HTTP infrastructure for this separate read path; it must not attach, resize, claim geometry, change subscription generation, or alter the live applied cursor. Keep concurrency low and visible demand explicit. +- Preserve the history view's stable row/turn anchor when inserting older pages. Live terminal output continues on its own surface; returning to live reveals its current screen. Alternate-screen TUI redraw streams are not a chronological transcript: expose only meaningful retained normal-buffer history or clearly state that earlier history is unavailable. +- The affordance says "Load earlier" only while a valid continuation exists. If the ring advances while browsing, show an expired boundary and stop requesting that cursor instead of triggering full reattachment. Tests: -- E2E: scroll to the top of a long terminal, load a chunk, repeat — history walks backwards without stalls, and the boundary is visible until fully loaded. +- E2E: enter history, load several bounded pages while the terminal produces live output, preserve the history scroll anchor, and return to an unchanged live stream identity and correct current screen. No `terminal.attach`, resize, or cursor reset is emitted by a history fetch. +- Integration: cursor expiry, stream replacement, Unicode/control-sequence boundaries, alternate-screen history unavailability, and oversized content all produce bounded, honest results with no repeated zero-progress fetches. ### Workstream 5 — Stop covering panes that have content (client + server) -Implementation notes: +Split this work into a refresh-correctness change and a subsequent pagination change. The former does not need to wait for a new transcript format. + +Refresh correctness: -- Replace the full-pane overlay in `FreshAgentView.tsx:3360`/`:3742` with an in-place refresh: render the last known conversation immediately and show a small status line while updating. -- Paginate the conversation snapshot endpoint (`/api/fresh-agent/threads/...`, currently 6 MB): recent turns first, older turns on demand, with a cursor. -- Hidden panes do not poll or fetch at all; on reveal, fetch the recent slice only. +- Replace the full-pane overlay in `FreshAgentView.tsx` with the last known conversation plus a small accessible refresh/error status and retry control. Preserve composer focus, selection, and scroll; an empty first load can still show a loading state. +- Fix the dirty-state condition, not just its presentation. Hidden reconnect calls `markSnapshotDirty`, capturing the current revision, but reveal currently accepts only a strictly greater revision (`FreshAgentView.tsx`, `markSnapshotDirty`, reconnect effect, and `revealRevisionIsFresh`). An unchanged idle conversation can therefore retry every 250 ms until error despite successful GETs. +- Track why a snapshot is dirty and the invalidation generation. For transport-only uncertainty, accept an authoritative same-revision response from a request started after that invalidation, with matching thread, owner, and request identity and no newer invalidation. A lower revision or response started before the invalidation cannot clear it. For a known mutation, require the advertised minimum revision or mutation-specific acknowledgement; do not globally replace `>` with `>=` and accidentally accept pre-mutation content. +- Retain the shared snapshot scheduler's single-flight, trailing refresh, and 429 backoff behavior. Hidden polling and event-triggered fetch suppression already exist; close the initial identity-fetch gap and add execution-time visibility demand to queued work. A hidden pane must not abort a shared request needed by a visible sibling. If no consumer remains visible, skip unstarted body fetches; a request already in flight may finish without scheduling follow-up work. +- Make freshness refer to actual scheduler execution, not just a caller's render/effect time. Preserve the existing rule that a request scheduled during an in-flight GET receives a trailing execution. Define visible-demand registration per snapshot key so the latest hidden caller cannot suppress work required by a visible consumer. Provider revisions are not one global sequence: audit each provider's revision and successful-response ownership metadata before defining mutation freshness, adding missing contract fields where necessary. +- Preserve lightweight fresh-agent attachment and status delivery for hidden panes: these support ownership, approvals, and completion notifications and are not transcript hydration. + +Pagination contract: + +- Reuse or extend `FreshAgentTurnPageSchema` and `FreshAgentTurnBodySchema` in `shared/fresh-agent-contract.ts` and the turns/body query and stale-revision schemas in `shared/read-models.ts`; their existence is not evidence of implemented Rust paging routes. `crates/freshell-freshagent/src/snapshot.rs` currently serves the full snapshot route. Define typed query validation and exhausted/stale/invalid-cursor responses, then update Rust responses, strict client schemas, API helpers, and consumers together. The existing whole-turn body schema cannot represent item previews or partial bodies without an explicit extension. +- Return current status, capabilities, approval/question state, revision, and a bounded recent-turn page. Older pages have stable turn/item IDs and a cursor scoped to thread identity and transcript revision. Explicitly report exhausted or invalidated cursors. Decide capability negotiation before changing the existing full-snapshot response consumed by older clients and the settings UI. +- Enforce serialized UTF-8 response-byte limits, including metadata and `rolledBackTurns`, not just a turn count. Large text, tool results, commands, attachments, and extension bodies need explicit previews plus lazy, bounded body/item pages. Never split JSON or pretend a truncated tool result is complete. A single oversized turn must still make progress through its body cursor. Keep approval/question semantics intact; if essential control metadata itself exceeds the supported budget, return an explicit bounded error instead of silently omitting controls. +- Replace whole-transcript replacement with revision-aware page storage. `mergeSnapshotForDisplay` currently replaces all turns on idle snapshots, which would discard loaded older pages. Define refresh page replacement, deduplication, and invalidation when revision changes; retain older pages only when continuity is established. Rollback/redo/fork must invalidate affected pages and bodies so a late response cannot resurrect removed turns. Treat rolled-back markers as separately paged history while retaining current redo metadata. +- Keep optimistic sends, send acknowledgements, and pending approvals separate from page membership. Absence from a partial recent page is not proof that a submitted turn was rejected or removed. Audit `localEchoLanded` / `shouldClearStaleLocalEcho` and the outgoing-turn completion path for that assumption. +- Bound server work as well as response size: prefer provider paging where available; otherwise reuse a bounded per-thread normalized index/cache. Do not rebuild and serialize a 6 MB transcript for every small page or every sibling pane. Measure provider-specific time and memory before claiming the one-second goal. Tests: -- Client: a revealed pane shows its last known content instantly while refreshing (no full-pane cover). -- Server: snapshot payloads are bounded and support fetching older turns. -- Client: hidden panes issue no requests. +- Hide → reconnect → reveal an unchanged idle conversation: one current same-revision response clears transport dirtiness; an older in-flight response or a response racing a new invalidation does not. +- A reveal scheduled while an earlier GET is in flight receives a fresh trailing execution. Hiding during debounce, 429 backoff, or an active shared request neither starts unnecessary follow-up work nor blocks a visible sibling. +- Refresh failure and 429 leave existing content usable; retry converges without a full-pane cover or focus/scroll loss. +- Hidden initial mount, poll, event, and queued refresh do not start body fetches; a visible sibling sharing the key still receives its response. Hidden status/completion/approval updates continue. +- Load older pages, then refresh while idle and while busy: no duplicates or unintended history loss; rollback/redo/fork followed by a late page/body response does not resurrect stale turns. +- A single multi-megabyte tool result, a large rolled-back history, and an optimistic turn outside the recent page exercise the real HTTP-to-client path. Assert bounded page/body bytes, honest continuation, and correct send state. ## Sequencing -1. Workstreams 1 and 2 together — the correctness floor: bounded restores that always finish. -2. Workstream 3 — small server change that removes the last way to get stuck. -3. Workstream 4 — the history-loading experience. -4. Workstream 5 — the agent-pane experience. +1. **Contract and reconstruction validation.** Specify baseline coverage, parser credit, capability negotiation, loss outcomes, and screen comparison fixtures. Prototype the current-screen representation and record supported/unsupported terminal state. Select the history representation before implementing workstream 4. A failed prototype is a design blocker for screen-first delivery, not permission to ship arbitrary tail reconstruction. +2. **Bounded recovery increment: workstreams 1 (paced path), 2, and 3 together.** Implement and test partial-hydrate checkpoints, removal of automatic OpenCode gap kills, bounded replay production, gap repair, and liveness limits as one coherent behavior change. A red base gate blocks implementation per repository rules. This increment can improve connection stability without yet meeting fresh-screen latency targets when no valid baseline exists. +3. **Screen-first increment: workstream 1 snapshot path.** Enable only after the reconstruction gate passes for the supported terminal modes and snapshot→delta handoff. Measure actual first-screen/input latency on the incident layout. A capability or supported-state fallback must remain explicit and non-destructive. +4. **History increment: workstream 4.** Implement its chosen content model and separate history view after the baseline machinery is established. Update `docs/index.html` for the significant UI change and README usage guidance. +5. **Conversation increments: workstream 5.** Refresh correctness and visibility scheduling can proceed independently of terminal work. Paginated transcripts follow their strict page/body/merge contract and provider validation; update the UI mock and README when the new history controls land. + +Use red/green/refactor for implementation, extending existing behavior tests rather than tests that assert plan text. Changes are authored in dedicated worktrees from a freshly checked green base; coordinate broad tests and use the configured backends. No PR creation or live deployment is authorized by this plan. Prepare and verify each increment, then obtain the user's explicit PR approval under repository rules. ## Acceptance criteria -End-user: +Correctness and resource limits (required for the bounded recovery increment): + +- Every rendered current screen is backed by a compatible applied checkpoint or validated reconstruction. Incomplete/unavailable state is labeled honestly; no healthy process is killed or replaced because replay is missing. +- Cutting the network mid-restore preserves the visible surface. A compatible partial restore requests only bytes after its parser-applied position; incompatible state takes the explicit baseline-recovery path. Retention expiry has a bounded failure outcome. +- Serialized batches, unacknowledged windows, per-connection restore state, writer queue/in-flight bytes, and metadata stay within declared limits. Large individual frames and snapshots have explicit continuation or bounded failure behavior, not silent over-budget sends. +- A slow, progressing consumer remains connected under terminal-output pressure; a dead socket still times out. Gap delivery and repair preserve per-terminal ordering and never advance checkpoints over missing parser input. +- Hidden panes start no screen/body hydration while hidden; necessary lifecycle and lightweight status subscriptions continue. A visibility change preserves terminal lifetime and other devices' geometry. +- Mixed-version tests prove new restore messages are negotiated and legacy OpenCode clients do not encounter newly introduced kill-triggering retention gaps. + +Final user experience (required before calling the entire plan complete): -- With the incident layout (7 terminals + 7 agent panes), the active pane is interactive in about a second; total startup traffic is a few hundred KB per pane; no disconnects; no multi-second freezes. -- Cutting the network mid-restore: the UI keeps showing what it had, and the restore completes after reconnect without resending everything. -- Scrolling to the top of a long terminal loads chunks without stalling and shows an honest boundary. -- No pane is ever covered by a full-screen refreshing curtain while it has content. +- On the incident layout (7 terminals + 7 agent panes), target p95 active-pane correct-screen and input-echo latency of approximately one second over repeated cold-client and reconnect trials on the incident hardware/network. Report trial count, terminal modes, retained bytes, and provider/snapshot sizes. Record main-thread long tasks and input latency during restore so "no multi-second freezes" is measured rather than inferred from payload size. +- Use 128 KiB as the initial terminal batch target, not a universal claim about total reconstruction size. Record total startup bytes separately from in-flight bytes; a full paced replay may legitimately exceed one batch, while a validated screen snapshot should avoid reading all history. Select aggregate window limits from measurements and publish them with the implementation. +- Earlier retained terminal history loads in bounded pages with a stable scroll anchor and an honest unloaded/expired boundary; live output continues independently. Returning to live shows the correct screen. +- Conversations with existing content remain usable during refresh, including same-revision reconnects and refresh failure. Recent pages and large bodies are byte-bounded; older loaded content, optimistic sends, and rollback state remain consistent. -Automated: +Use a deterministic fixture matching the incident's data sizes for repeatable integration/browser tests, then a manually observed Windows Electron smoke check. Validate affected browser specs on the configured backend and ensure they actually execute; destructive restart/kill test cases belong in the disposable sandbox. Do not experiment on live user terminals to manufacture loss. -- Per-pane attach payloads stay within the requested budget; a budget gap is reported. -- A slow consumer receives gaps, never a disconnect; a dead consumer is still disconnected. -- A retry after a mid-restore drop requests only the remainder. -- Hidden panes neither attach nor poll. +Observability: add structured events for restore start/completion/failure, baseline source, requested/applied sequence, raw versus serialized bytes, unacknowledged bytes, queue/in-flight pressure, gap reason, successful-send progress, retries, and first-screen/input timing. For conversation refreshes include trigger, execution generation, revision acceptance reason, page bytes, and visible demand. Log identifiers and measurements, not terminal or conversation contents. Keep diagnostic logging lightweight and debug perf logging off outside investigations. ## Risks and open questions -- Alt-screen reconstruction: a small slice of a full-screen app must reliably rebuild the visible screen. Validate on opencode, Codex, and Claude TUIs; keep the repaint nudge as the fallback if a quiet pane's newest slice does not cover the screen. -- Budget defaults: 128 KB for visible panes; decide the hidden-pane and per-connection totals. -- History fetch transport: new WebSocket message versus an HTTP endpoint; pick during implementation. -- Retry ceiling: how many attempts before showing the explicit failure state. -- Deployment: server-side changes require a restart of the live 3.150 server, which needs the user's explicit approval. -- Base health: the coordinator's latest full-suite run is red for reasons unrelated to this document; re-check the base before implementing. +- **Screen reconstruction gate:** choose and validate the emulator/checkpoint format, state coverage, initialization for already-running streams, and maximum snapshot size. This is the main unresolved design decision; mode flags and resize nudges are not substitutes. If it cannot meet fidelity and latency requirements, revise the screen-first design before implementation of that increment. +- **History representation gate:** choose bounded normal-buffer rows or independently reconstructible pages. Define scroll anchors and retention cost; do not claim arbitrary TUI redraw history is a transcript. +- **Protocol details:** finalize parser-credit window, oversized-frame handling, atomic baseline installation, capability names, and compatibility behavior. Preserve the existing `maxReplayBytes` meaning and distinguish raw payload from wire accounting. +- **Limits and retries:** select aggregate byte/metadata limits and recovery-attempt/no-progress deadlines from fixture measurements. Define explicit behavior when output production outruns finite retention; "always finishes" is not a defensible unconditional promise. +- **Provider pagination:** provider revision semantics and paging support differ. Validate stale-page handling, large-body access, pending-send reconciliation, ownership changes, and full-snapshot callers before enabling partial responses. +- **Deployment:** server changes require the approved self-hosted launch runbook and an explicit `APPROVED` before restarting the live server. Neither this plan edit nor test-instance permission authorizes that restart. +- **Base health:** the original investigation reported a red latest full-suite result. That is historical, not a current gate result; re-check `scripts/base-gate.sh` before creating an implementation worktree. This documentation revision does not require a production restart or a broad test run. ## Appendix — incident log excerpts @@ -207,4 +270,4 @@ INFO ws.connection.established ... repeating every 15-20 s ``` -Client console during the loop: `xterm.js: Parsing error` entries while re-parsing replay data on each reconnect; the app remained connected but could not finish restoring. +Client console during the loop: `xterm.js: Parsing error` entries while re-parsing replay data on each reconnect; the app remained open but could not finish restoring. From a5073890db8ec775cc2c0d35e8269918d7ab7e5c Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:02:19 -0700 Subject: [PATCH 03/70] docs(plans): carry the current user request block for the restore implementation run --- .../2026-09-19-responsive-terminal-restore.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/plans/2026-09-19-responsive-terminal-restore.md b/docs/plans/2026-09-19-responsive-terminal-restore.md index d2a00467d..ad706de23 100644 --- a/docs/plans/2026-09-19-responsive-terminal-restore.md +++ b/docs/plans/2026-09-19-responsive-terminal-restore.md @@ -5,6 +5,22 @@ Date: 2026-09-19 Incident: live self-hosted server on garageserver (`192.168.3.150:3001`), Electron client on DANDESKTOP Related: `docs/plans/2026-03-30-paginated-terminal-replay.md`, `docs/plans/2026-07-24-rust-attach-viewport.md` (TERM-07 open items) +## User Request + +### Requested result +- Implement the committed plan `docs/plans/2026-09-19-responsive-terminal-restore.md` (reliable, responsive pane restore) starting with its own Sequencing: the shared restore contract plus the bounded recovery increment (workstreams 1 paced path, 2, and 3 together), fully tested and prepared for PR approval. + +### Explicit constraints +- Follow the plan's Sequencing section: the shared restore contract and bounded recovery increment come first; the screen-first snapshot path, on-demand history, and conversation refresh/pagination are later increments gated on their design validations and are not in this run. +- Use red/green/refactor TDD; extend existing behavior tests rather than tests that assert plan text. +- No PR creation and no live deployment; never restart the live self-hosted server without the user's explicit "APPROVED". +- Broad repo-supported test runs go through the shared coordinator gate; the worktree base must be green via `scripts/base-gate.sh`. +- Do not experiment on live user terminals to manufacture loss; use disposable test instances and fixtures. +- Older clients must not receive newly introduced retention gaps; capability negotiation gates all new restore semantics, and no healthy process is killed or replaced because replay is missing. + +### Accepted tradeoffs and residuals +- The bounded recovery increment may not meet the one-second fresh-screen latency target when no valid baseline exists; screen-first snapshots, on-demand history loading, and conversation refresh/pagination correctness remain follow-on increments per the plan's Sequencing. + ## Summary When a client attaches to a terminal, the server sends its entire retained output newer than the requested position and ignores the requested replay budget. In the incident, initial and repeated full hydrations amounted to roughly 21 MB across seven terminals. The server's sustained-backlog disconnect threshold is lower than its output-spill threshold, creating a reconnect loop. The client already records parser-applied progress, but an incomplete fresh-surface hydrate and other safety checks can force the next attempt back to zero. From a0f5a4d73751dc8dc9efe12f5fcada239374d18c Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:02:19 -0700 Subject: [PATCH 04/70] docs(plans): require a negotiated hidden-pane lifetime claim in the restore contract --- docs/plans/2026-09-19-responsive-terminal-restore.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plans/2026-09-19-responsive-terminal-restore.md b/docs/plans/2026-09-19-responsive-terminal-restore.md index ad706de23..1d2edc797 100644 --- a/docs/plans/2026-09-19-responsive-terminal-restore.md +++ b/docs/plans/2026-09-19-responsive-terminal-restore.md @@ -145,11 +145,11 @@ Implementation requirements: - Bound both per-pane unacknowledged replay and total per-connection admitted bytes, including snapshot/replay copies and the frame being sent. Select pages before cloning payloads; do not first allocate one full replay copy per pane. Cancellation, superseded attaches, and disconnect release credits and temporary state. - Do not pin an unbounded replay backlog for a slow client. If retention overtakes a continuation cursor, report the exact lost interval and enter bounded baseline recovery. A finite retention window cannot guarantee convergence against indefinitely faster output production. - Remove the proposed resize nudge. Attach resize and replay capture currently share the terminal lock needed by the output reader; an immediate capture cannot include the asynchronous repaint, and waiting under that lock prevents ingestion. Same-size resize is a no-op, and another subscriber may own geometry. Any future repaint protocol must await a verifiable result outside that lock, respect geometry ownership, and fall back on timeout; it is not a prerequisite for the paced path. -- Hidden panes do not request screen hydration until revealed. Preserve cheap lifecycle/status subscriptions separately from output delivery and terminal lifetime. Do not use ordinary `terminal.detach` as a visibility toggle without checking its release/reap semantics. Remove hidden hydration-queue grants rather than sending a nominal zero budget that the current truthy serializer omits. On reveal, use a valid retained surface checkpoint or request a fresh baseline. +- Hidden panes do not request screen hydration until revealed. Preserving terminal lifetime is a server-side claim, not an inference: `TerminalShared::released_by_client` starts `true` and only a successful attach clears it, and the idle reaper (`enforce_idle_kills`) applies the configured idle threshold only to `released_by_client` rows — never-attached hidden panes survive today only because hidden hydration attaches them. Removing hidden hydration therefore requires a negotiated, non-hydrating per-connection lifetime claim (additive optional field on the already-negotiated `terminal.interest` presentation snapshot, e.g. a `claimedTerminalIds` set) that marks claimed terminals wanted with the same release semantics as an explicit attach: the claim drops when its connection — the last claimer or subscriber — goes away, never grants replay or output delivery, and does not touch geometry or stream identity. A client whose server did not negotiate the claim field keeps today's hidden attach (`keepalive_delta`) as its lifetime claim — compatibility outranks the hidden-pane optimization. Preserve cheap lifecycle/status subscriptions separately from output delivery and terminal lifetime. Do not use ordinary `terminal.detach` as a visibility toggle without checking its release/reap semantics. Remove hidden hydration-queue grants rather than sending a nominal zero budget that the current truthy serializer omits. On reveal, use a valid retained surface checkpoint or request a fresh baseline. Tests: -- Registry/WS: byte budgets, continuation bounds, fixed catch-up target, snapshot→live ordering under concurrent output, expired continuation, cancellation, and bounded aggregate memory with many panes. +- Registry/WS: byte budgets, continuation bounds, fixed catch-up target, snapshot→live ordering under concurrent output, expired continuation, cancellation, and bounded aggregate memory with many panes. Hidden-pane lifetime: a created-hidden and a reconnect-hidden terminal held only by the negotiated lifetime claim survives the configured idle-kill threshold (the claim clears `released_by_client` without attaching or delivering replay), and dropping the last claim re-exposes the terminal to that threshold. - Client/WS: withhold parser callbacks to prove replay credit does not advance on receipt; release callbacks and prove forward progress without duplicate output or same-terminal reordering. - Reconstruction: compare visible cells, cursor, attributes, modes, and subsequent input behavior against an uninterrupted xterm reference using shell, OpenCode, Codex, and Claude recordings, plus real TUI smoke tests. Include quiet/no-repaint output, split control sequences, alternate-buffer transitions, different geometry, second viewers, and screen representations larger than the nominal budget. - Compatibility: new/old client-server pairs select supported behavior; lack of capability never causes an automatic kill or claims a raw suffix is a complete screen. From 87e83028f7d0bf3bb77505c0d03510b98e135e88 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:14:13 -0700 Subject: [PATCH 05/70] docs(plans): split continuation consumption credit from the parser-applied checkpoint and scope the increment's hidden-pane acceptance to terminal screens --- docs/plans/2026-09-19-responsive-terminal-restore.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/plans/2026-09-19-responsive-terminal-restore.md b/docs/plans/2026-09-19-responsive-terminal-restore.md index 1d2edc797..308974fe2 100644 --- a/docs/plans/2026-09-19-responsive-terminal-restore.md +++ b/docs/plans/2026-09-19-responsive-terminal-restore.md @@ -133,7 +133,7 @@ Evidence for these constraints: `TerminalView.tsx` (`completeParserAppliedFrame` Deliver in two increments. Paced replay fixes the unbounded burst; a screen snapshot is needed to make fresh TUI restoration independent of history size. The first increment must not be presented as meeting the final one-second screen-first goal. -1. **Paced forward restoration.** Negotiate a continuation/credit capability (proposed name: `pacedTerminalReplayV1`) that sends bounded, ascending sequence batches from a compatible applied cursor or a proven fresh-surface baseline. Parser acknowledgements carry terminal, stream, attach generation, and the last fully applied sequence; stale acknowledgements or values beyond the sent window cannot grant credit. Grant more replay credit only after xterm applies the prior batch. Keep a fixed initial catch-up target so ongoing live output cannot move the completion condition indefinitely; subsequently deliver live output in sequence order. Do not let live frames for the same terminal overtake unapplied replay. Input and other panes' traffic remain schedulable. +1. **Paced forward restoration.** Negotiate a continuation/credit capability (proposed name: `pacedTerminalReplayV1`) that sends bounded, ascending sequence batches from a compatible applied cursor or a proven fresh-surface baseline. Continuation acknowledgements carry terminal, stream, attach generation, and the last fully consumed sequence; stale acknowledgements or values beyond the sent window cannot grant credit. Consumption credit and the reconstructible parser-applied checkpoint are distinct and must not be conflated: the client's ordered write queue consumes every frame — each frame is either applied by xterm or deliberately filtered by a byte-mutating parser (startup probes, OSC52, completion signals) that by design never reaches xterm, and a filtered frame must not stall continuation. Credit is granted only after the prior batch is consumed in order; the parser-applied checkpoint keeps its strict semantics and never advances across a filtered or lost range. Tests must cover a page containing only filtered frames and a mixed page: both credit continuation, while the checkpoint stays pinned at its pre-filter position. Keep a fixed initial catch-up target so ongoing live output cannot move the completion condition indefinitely; subsequently deliver live output in sequence order. Do not let live frames for the same terminal overtake unconsumed replay. Input and other panes' traffic remain schedulable. 2. **Validated current-screen restoration.** Prototype a server-maintained terminal emulator/checkpoint, updated in PTY output order, that can reconstruct both normal and alternate buffers plus cursor, attributes, modes, and geometry. A snapshot carries its stream/geometry identity and covered sequence; snapshot capture and subscription installation must establish a gap-free snapshot→delta handoff. Compare its output against the real client parser before selecting the implementation. Do not add an emulator dependency solely on the assumption that it is xterm-compatible. The paced path can reconstruct exactly only if its starting state and required bytes are available. The retained ring may already have lost essential state, so replaying its entire remaining tail is not a proof of correctness either. In that case use a validated snapshot when available, or show an explicit incomplete-screen state with retry; do not restart the program automatically. A newly introduced server emulator cannot retroactively reconstruct missing history for an already-running stream: mark its initial baseline unknown until a verified reset/repaint boundary, or use the explicit unavailable state. @@ -178,7 +178,7 @@ Tests: Implementation notes: - Set consistent byte limits so normal output pressure reaches bounded admission/spill before any pressure-related disconnect. Include the leased in-flight frame; `DeliveryQueue::outstanding_bytes` already charges reserved bytes. Keep independent hard bounds on metadata/control mailboxes. Validate environment overrides so a configuration cannot silently restore the 32 MB spill / 16 MB disconnect mismatch. -- Replace the sustained-byte-count-only disconnect decision for healthy draining clients. Preserve the existing independent socket-send timeout and keepalive failure handling; slow consumption alone is not a dead socket. If tracking drain progress, count successful sends, not falling queue size: eviction and superseded attachments also reduce queue bytes. Parser credit from workstream 1 governs restore production independently of socket liveness. +- Replace the sustained-byte-count-only disconnect decision for healthy draining clients. Preserve the existing independent socket-send timeout and keepalive failure handling; slow consumption alone is not a dead socket. If tracking drain progress, count successful sends, not falling queue size: eviction and superseded attachments also reduce queue bytes. Continuation credit from workstream 1 governs restore production independently of socket liveness. - Keep ordered, generation-scoped overflow gaps and non-evictable sequenced controls. A spill bounds memory but does not repair a terminal screen: route the loss through workstream 2's repair contract. Do not exempt replay from memory accounting or silently drop control frames. - Reuse existing focused/visible/background byte-fair scheduling. Keep input, completion/status, and other panes responsive while a terminal restores; avoid a second competing priority scheduler. @@ -237,7 +237,7 @@ Tests: ## Sequencing -1. **Contract and reconstruction validation.** Specify baseline coverage, parser credit, capability negotiation, loss outcomes, and screen comparison fixtures. Prototype the current-screen representation and record supported/unsupported terminal state. Select the history representation before implementing workstream 4. A failed prototype is a design blocker for screen-first delivery, not permission to ship arbitrary tail reconstruction. +1. **Contract and reconstruction validation.** Specify baseline coverage, continuation credit, capability negotiation, loss outcomes, and screen comparison fixtures. Prototype the current-screen representation and record supported/unsupported terminal state. Select the history representation before implementing workstream 4. A failed prototype is a design blocker for screen-first delivery, not permission to ship arbitrary tail reconstruction. 2. **Bounded recovery increment: workstreams 1 (paced path), 2, and 3 together.** Implement and test partial-hydrate checkpoints, removal of automatic OpenCode gap kills, bounded replay production, gap repair, and liveness limits as one coherent behavior change. A red base gate blocks implementation per repository rules. This increment can improve connection stability without yet meeting fresh-screen latency targets when no valid baseline exists. 3. **Screen-first increment: workstream 1 snapshot path.** Enable only after the reconstruction gate passes for the supported terminal modes and snapshot→delta handoff. Measure actual first-screen/input latency on the incident layout. A capability or supported-state fallback must remain explicit and non-destructive. 4. **History increment: workstream 4.** Implement its chosen content model and separate history view after the baseline machinery is established. Update `docs/index.html` for the significant UI change and README usage guidance. @@ -253,7 +253,7 @@ Correctness and resource limits (required for the bounded recovery increment): - Cutting the network mid-restore preserves the visible surface. A compatible partial restore requests only bytes after its parser-applied position; incompatible state takes the explicit baseline-recovery path. Retention expiry has a bounded failure outcome. - Serialized batches, unacknowledged windows, per-connection restore state, writer queue/in-flight bytes, and metadata stay within declared limits. Large individual frames and snapshots have explicit continuation or bounded failure behavior, not silent over-budget sends. - A slow, progressing consumer remains connected under terminal-output pressure; a dead socket still times out. Gap delivery and repair preserve per-terminal ordering and never advance checkpoints over missing parser input. -- Hidden panes start no screen/body hydration while hidden; necessary lifecycle and lightweight status subscriptions continue. A visibility change preserves terminal lifetime and other devices' geometry. +- Hidden panes start no terminal screen hydration while hidden; necessary lifecycle, ownership, and lightweight status subscriptions continue. Conversation body-fetch visibility belongs to workstream 5 and is not this increment's acceptance. A visibility change preserves terminal lifetime and other devices' geometry. - Mixed-version tests prove new restore messages are negotiated and legacy OpenCode clients do not encounter newly introduced kill-triggering retention gaps. Final user experience (required before calling the entire plan complete): @@ -271,7 +271,7 @@ Observability: add structured events for restore start/completion/failure, basel - **Screen reconstruction gate:** choose and validate the emulator/checkpoint format, state coverage, initialization for already-running streams, and maximum snapshot size. This is the main unresolved design decision; mode flags and resize nudges are not substitutes. If it cannot meet fidelity and latency requirements, revise the screen-first design before implementation of that increment. - **History representation gate:** choose bounded normal-buffer rows or independently reconstructible pages. Define scroll anchors and retention cost; do not claim arbitrary TUI redraw history is a transcript. -- **Protocol details:** finalize parser-credit window, oversized-frame handling, atomic baseline installation, capability names, and compatibility behavior. Preserve the existing `maxReplayBytes` meaning and distinguish raw payload from wire accounting. +- **Protocol details:** finalize the continuation-credit window (consumption credit distinct from the parser-applied checkpoint), oversized-frame handling, atomic baseline installation, capability names, and compatibility behavior. Preserve the existing `maxReplayBytes` meaning and distinguish raw payload from wire accounting. - **Limits and retries:** select aggregate byte/metadata limits and recovery-attempt/no-progress deadlines from fixture measurements. Define explicit behavior when output production outruns finite retention; "always finishes" is not a defensible unconditional promise. - **Provider pagination:** provider revision semantics and paging support differ. Validate stale-page handling, large-body access, pending-send reconciliation, ownership changes, and full-snapshot callers before enabling partial responses. - **Deployment:** server changes require the approved self-hosted launch runbook and an explicit `APPROVED` before restarting the live server. Neither this plan edit nor test-instance permission authorizes that restart. From 362d2c3af4757e6a58b9bfb89ad7fb17ec4ab0e0 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:37:57 -0700 Subject: [PATCH 06/70] docs(plans): add the surface-coverage resume cursor and attach-exact lifetime-claim release semantics --- docs/plans/2026-09-19-responsive-terminal-restore.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/plans/2026-09-19-responsive-terminal-restore.md b/docs/plans/2026-09-19-responsive-terminal-restore.md index 308974fe2..c87362a09 100644 --- a/docs/plans/2026-09-19-responsive-terminal-restore.md +++ b/docs/plans/2026-09-19-responsive-terminal-restore.md @@ -133,7 +133,7 @@ Evidence for these constraints: `TerminalView.tsx` (`completeParserAppliedFrame` Deliver in two increments. Paced replay fixes the unbounded burst; a screen snapshot is needed to make fresh TUI restoration independent of history size. The first increment must not be presented as meeting the final one-second screen-first goal. -1. **Paced forward restoration.** Negotiate a continuation/credit capability (proposed name: `pacedTerminalReplayV1`) that sends bounded, ascending sequence batches from a compatible applied cursor or a proven fresh-surface baseline. Continuation acknowledgements carry terminal, stream, attach generation, and the last fully consumed sequence; stale acknowledgements or values beyond the sent window cannot grant credit. Consumption credit and the reconstructible parser-applied checkpoint are distinct and must not be conflated: the client's ordered write queue consumes every frame — each frame is either applied by xterm or deliberately filtered by a byte-mutating parser (startup probes, OSC52, completion signals) that by design never reaches xterm, and a filtered frame must not stall continuation. Credit is granted only after the prior batch is consumed in order; the parser-applied checkpoint keeps its strict semantics and never advances across a filtered or lost range. Tests must cover a page containing only filtered frames and a mixed page: both credit continuation, while the checkpoint stays pinned at its pre-filter position. Keep a fixed initial catch-up target so ongoing live output cannot move the completion condition indefinitely; subsequently deliver live output in sequence order. Do not let live frames for the same terminal overtake unconsumed replay. Input and other panes' traffic remain schedulable. +1. **Paced forward restoration.** Negotiate a continuation/credit capability (proposed name: `pacedTerminalReplayV1`) that sends bounded, ascending sequence batches from a compatible applied cursor or a proven fresh-surface baseline. Continuation acknowledgements carry terminal, stream, attach generation, and the last fully consumed sequence; stale acknowledgements or values beyond the sent window cannot grant credit. Consumption credit and the reconstructible parser-applied checkpoint are distinct and must not be conflated: the client's ordered write queue consumes every frame — each frame is either applied by xterm or deliberately filtered by a byte-mutating parser (startup probes, OSC52, completion signals) that by design never reaches xterm, and a filtered frame must not stall continuation. Credit is granted only after the prior batch is consumed in order; the parser-applied checkpoint keeps its strict semantics and never advances across a filtered or lost range. A pinned checkpoint must not strand an interrupted restore: define a reconstruction-safe surface-coverage cursor, separate from the strict parser-applied sequence, that advances past a filtered range only when the applied filter class has a proven null screen effect (startup-probe suppression, OSC52 handling, completion-signal consumption); any unknown mutation leaves the coverage cursor pinned and keeps the quarantine path. Delta resumes and partial-hydrate continuation request their since position from the coverage cursor, so an interrupted restore after filtered or mixed pages neither duplicates already-rendered output nor forces a full baseline rebuild, and the checkpoint carries the coverage cursor through the same identity/geometry/authority validation as the applied sequence. Tests must cover a page containing only filtered frames and a mixed page — both credit continuation, the checkpoint stays pinned at its pre-filter position — plus disconnect-and-resume across both page shapes: the resumed surface converges to the uninterrupted reference with no duplicate writes and no spurious baseline recovery. Keep a fixed initial catch-up target so ongoing live output cannot move the completion condition indefinitely; subsequently deliver live output in sequence order. Do not let live frames for the same terminal overtake unconsumed replay. Input and other panes' traffic remain schedulable. 2. **Validated current-screen restoration.** Prototype a server-maintained terminal emulator/checkpoint, updated in PTY output order, that can reconstruct both normal and alternate buffers plus cursor, attributes, modes, and geometry. A snapshot carries its stream/geometry identity and covered sequence; snapshot capture and subscription installation must establish a gap-free snapshot→delta handoff. Compare its output against the real client parser before selecting the implementation. Do not add an emulator dependency solely on the assumption that it is xterm-compatible. The paced path can reconstruct exactly only if its starting state and required bytes are available. The retained ring may already have lost essential state, so replaying its entire remaining tail is not a proof of correctness either. In that case use a validated snapshot when available, or show an explicit incomplete-screen state with retry; do not restart the program automatically. A newly introduced server emulator cannot retroactively reconstruct missing history for an already-running stream: mark its initial baseline unknown until a verified reset/repaint boundary, or use the explicit unavailable state. @@ -145,11 +145,11 @@ Implementation requirements: - Bound both per-pane unacknowledged replay and total per-connection admitted bytes, including snapshot/replay copies and the frame being sent. Select pages before cloning payloads; do not first allocate one full replay copy per pane. Cancellation, superseded attaches, and disconnect release credits and temporary state. - Do not pin an unbounded replay backlog for a slow client. If retention overtakes a continuation cursor, report the exact lost interval and enter bounded baseline recovery. A finite retention window cannot guarantee convergence against indefinitely faster output production. - Remove the proposed resize nudge. Attach resize and replay capture currently share the terminal lock needed by the output reader; an immediate capture cannot include the asynchronous repaint, and waiting under that lock prevents ingestion. Same-size resize is a no-op, and another subscriber may own geometry. Any future repaint protocol must await a verifiable result outside that lock, respect geometry ownership, and fall back on timeout; it is not a prerequisite for the paced path. -- Hidden panes do not request screen hydration until revealed. Preserving terminal lifetime is a server-side claim, not an inference: `TerminalShared::released_by_client` starts `true` and only a successful attach clears it, and the idle reaper (`enforce_idle_kills`) applies the configured idle threshold only to `released_by_client` rows — never-attached hidden panes survive today only because hidden hydration attaches them. Removing hidden hydration therefore requires a negotiated, non-hydrating per-connection lifetime claim (additive optional field on the already-negotiated `terminal.interest` presentation snapshot, e.g. a `claimedTerminalIds` set) that marks claimed terminals wanted with the same release semantics as an explicit attach: the claim drops when its connection — the last claimer or subscriber — goes away, never grants replay or output delivery, and does not touch geometry or stream identity. A client whose server did not negotiate the claim field keeps today's hidden attach (`keepalive_delta`) as its lifetime claim — compatibility outranks the hidden-pane optimization. Preserve cheap lifecycle/status subscriptions separately from output delivery and terminal lifetime. Do not use ordinary `terminal.detach` as a visibility toggle without checking its release/reap semantics. Remove hidden hydration-queue grants rather than sending a nominal zero budget that the current truthy serializer omits. On reveal, use a valid retained surface checkpoint or request a fresh baseline. +- Hidden panes do not request screen hydration until revealed. Preserving terminal lifetime is a server-side claim, not an inference: `TerminalShared::released_by_client` starts `true` and only a successful attach clears it, and the idle reaper (`enforce_idle_kills`) applies the configured idle threshold only to `released_by_client` rows — never-attached hidden panes survive today only because hidden hydration attaches them. Removing hidden hydration therefore requires a negotiated, non-hydrating per-connection lifetime claim (additive optional field on the already-negotiated `terminal.interest` presentation snapshot, e.g. a `claimedTerminalIds` set) that marks claimed terminals wanted with release semantics mirroring attach exactly: claim membership is per-connection and sweeps away with its socket, but transport loss is not release — a dropped connection must keep the durable wanted state, exactly as `remove_connection` never restores fast reapability today and only an explicit `terminal.detach` of the last reference does. The only path back to configured-threshold reapability is an explicit release action: claim withdrawal, or the existing detach reconciler acting when the terminal leaves every pane layout. The claim never grants replay or output delivery and does not touch geometry or stream identity. A client whose server did not negotiate the claim field keeps today's hidden attach (`keepalive_delta`) as its lifetime claim — compatibility outranks the hidden-pane optimization. Preserve cheap lifecycle/status subscriptions separately from output delivery and terminal lifetime. Do not use ordinary `terminal.detach` as a visibility toggle without checking its release/reap semantics. Remove hidden hydration-queue grants rather than sending a nominal zero budget that the current truthy serializer omits. On reveal, use a valid retained surface checkpoint or request a fresh baseline. Tests: -- Registry/WS: byte budgets, continuation bounds, fixed catch-up target, snapshot→live ordering under concurrent output, expired continuation, cancellation, and bounded aggregate memory with many panes. Hidden-pane lifetime: a created-hidden and a reconnect-hidden terminal held only by the negotiated lifetime claim survives the configured idle-kill threshold (the claim clears `released_by_client` without attaching or delivering replay), and dropping the last claim re-exposes the terminal to that threshold. +- Registry/WS: byte budgets, continuation bounds, fixed catch-up target, snapshot→live ordering under concurrent output, expired continuation, cancellation, and bounded aggregate memory with many panes. Hidden-pane lifetime: a created-hidden and a reconnect-hidden terminal held only by the negotiated lifetime claim survives the configured idle-kill threshold (the claim clears `released_by_client` without attaching or delivering replay); a claim-connection drop (transport loss) keeps the terminal wanted like any attached-then-disconnected terminal (24-hour hard cap only); an explicit claim withdrawal or detach of the last reference re-exposes the terminal to the configured threshold. - Client/WS: withhold parser callbacks to prove replay credit does not advance on receipt; release callbacks and prove forward progress without duplicate output or same-terminal reordering. - Reconstruction: compare visible cells, cursor, attributes, modes, and subsequent input behavior against an uninterrupted xterm reference using shell, OpenCode, Codex, and Claude recordings, plus real TUI smoke tests. Include quiet/no-repaint output, split control sequences, alternate-buffer transitions, different geometry, second viewers, and screen representations larger than the nominal budget. - Compatibility: new/old client-server pairs select supported behavior; lack of capability never causes an automatic kill or claims a raw suffix is a complete screen. @@ -159,7 +159,7 @@ Tests: Implementation notes: - Reuse the existing parser-applied checkpoint machinery. `completeParserAppliedFrame` saves progress after xterm's write callback; receipt/highest-observed sequence is not sufficient. The defect is that `surfaceFreshRef` can override an otherwise usable partial checkpoint and force zero replay, not that progress recording is entirely absent. -- Represent incomplete hydration separately from a newly created blank surface. A compatible partial hydrate can resume on the same mounted xterm without clearing it, replaying a mode preamble, or claiming `surfaceReset` again. Applying a complete validated snapshot establishes its covered-sequence baseline only after the parser applies it; incomplete snapshot installation must not be mistaken for resumable terminal deltas. +- Represent incomplete hydration separately from a newly created blank surface. A compatible partial hydrate can resume on the same mounted xterm without clearing it, replaying a mode preamble, or claiming `surfaceReset` again. After filtered or mixed pages the resume position comes from Workstream 1's surface-coverage cursor, never from a checkpoint the filter pinned below already-rendered content. Applying a complete validated snapshot establishes its covered-sequence baseline only after the parser applies it; incomplete snapshot installation must not be mistaken for resumable terminal deltas. - Retain `canUseCheckpointForDeltaReplay` checks for terminal/stream/server identity, surface generation, geometry and authority, scrollback, parser readiness, and xterm version. Recreated surfaces, changed streams, or incompatible geometry need a new valid baseline, even if the old sequence is nonzero. Scope checkpoints to the actual surface so sibling panes showing the same terminal cannot borrow each other's rendered progress. - When writes are in flight, freeze generation transitions and wait boundedly for the owned queue to drain before deciding whether its applied checkpoint remains usable. Do not clear a surface while an earlier write can still mutate it. If current quarantine rules have already invalidated that generation, preserve quarantine and rebuild from a valid baseline rather than accepting stale callbacks. - Implement reason-aware gaps from the shared contract. In particular, remove the `replay_window_exceeded` → `beginOpenCodeReplacementAfterExit` automatic kill path before enabling new server retention-gap notifications. Restore gaps must not emit `terminal.kill`, change terminal identity, or spawn a replacement for a healthy process. Any explicit restart action remains separate user intent. @@ -167,7 +167,7 @@ Implementation notes: Tests: -- Interrupt a hydrate after some xterm callbacks: same surface requests only the remainder, without clear/reset/preamble, and converges to the uninterrupted reference. +- Interrupt a hydrate after some xterm callbacks: same surface requests only the remainder, without clear/reset/preamble, and converges to the uninterrupted reference. Include disconnect-and-resume across a filtered-only page and a mixed page (Workstream 1's coverage-cursor case): no duplicate writes, no spurious baseline recovery, convergence to the reference. - Interrupt before a callback, delay old callbacks across recovery, and independently change each checkpoint identity/geometry field: no false progress, duplicate writes, or stale checkpoint acceptance. Page reload and sibling surfaces cannot reuse a cursor without their corresponding state. - Start with a validated bounded screen and unloaded history, apply live output, reconnect: the applied cursor advances while the history boundary remains. A true delivery gap still blocks unsafe advancement. - A healthy idle OpenCode terminal with expired retained output reports incomplete history/screen state as appropriate; terminal ID and process remain unchanged and subsequent live output still arrives. Repeat for a queue gap; separately verify the compatible path for an older unnegotiated client. From 9aa59156bbde924eb3eaf6342b174a16853754d8 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sat, 19 Sep 2026 23:20:24 -0700 Subject: [PATCH 07/70] feat(ws): negotiate the pacedTerminalReplayV1 capability Responsive-terminal-restore Workstream 1 negotiation rail (additive optional, no protocolVersion bump): the hello opt-in and ready echo for paced terminal replay, threaded hello -> ready gate -> terminal::run -> run_loop -> handle_client_text -> handle_attach -> registry attach, and parked on the attach subscriber alongside terminal_output_batch_v1 for the task-3 paced replay core. A hello without the capability stays byte-identical on both sides (frozen-client inertness), pinned by protocol roundtrips, in-file handshake tests, a real-socket e2e suite, and the client allowlist/lifecycle tests. Contract artifacts regenerated in lockstep. --- .../freshell-protocol/src/client_messages.rs | 6 + .../freshell-protocol/src/server_messages.rs | 6 + .../freshell-protocol/tests/pane_reconcile.rs | 1 + crates/freshell-protocol/tests/roundtrip.rs | 63 ++++ crates/freshell-terminal/src/registry.rs | 290 +++++++++++++++--- crates/freshell-ws/src/lib.rs | 56 +++- crates/freshell-ws/src/terminal.rs | 23 ++ .../freshell-ws/tests/hello_capabilities.rs | 208 +++++++++++++ port/contract/ws-protocol.schema.json | 12 + port/contract/ws-server-messages.schema.json | 3 + shared/ws-protocol.ts | 8 + src/lib/ws-client.ts | 2 +- test/unit/client/lib/pane-reconcile.test.ts | 10 + .../client/lib/ws-client.reconcile.test.ts | 22 +- test/unit/client/lib/ws-client.test.ts | 1 + 15 files changed, 665 insertions(+), 46 deletions(-) create mode 100644 crates/freshell-ws/tests/hello_capabilities.rs diff --git a/crates/freshell-protocol/src/client_messages.rs b/crates/freshell-protocol/src/client_messages.rs index 33ddd8008..109032bce 100644 --- a/crates/freshell-protocol/src/client_messages.rs +++ b/crates/freshell-protocol/src/client_messages.rs @@ -187,6 +187,12 @@ pub struct HelloCapabilities { /// advertises the capability back (§4.2). Absent for the frozen client. #[serde(skip_serializing_if = "Option::is_none")] pub pane_reconcile_v1: Option, + /// Paced terminal restore opt-in (responsive-terminal-restore Workstream + /// 1): the client understands bounded, ascending paced replay batches with + /// continuation credit. Additive optional — absent on the frozen client + /// and stripped-tolerant on older servers (no version bump). + #[serde(skip_serializing_if = "Option::is_none")] + pub paced_terminal_replay_v1: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/crates/freshell-protocol/src/server_messages.rs b/crates/freshell-protocol/src/server_messages.rs index 1920932fc..8400321dc 100644 --- a/crates/freshell-protocol/src/server_messages.rs +++ b/crates/freshell-protocol/src/server_messages.rs @@ -936,6 +936,12 @@ pub struct ReadyCapabilities { pub pane_reconcile_fresh_agent_v1: Option, #[serde(skip_serializing_if = "Option::is_none")] pub terminal_interest_v1: Option, + /// Paced terminal restore (responsive-terminal-restore Workstream 1): + /// `Some(true)` iff the connection's `hello` opted in via + /// `capabilities.pacedTerminalReplayV1` — omitted from the wire entirely + /// otherwise (frozen-client inertness). + #[serde(skip_serializing_if = "Option::is_none")] + pub paced_terminal_replay_v1: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/crates/freshell-protocol/tests/pane_reconcile.rs b/crates/freshell-protocol/tests/pane_reconcile.rs index 804985adf..46f78ee9e 100644 --- a/crates/freshell-protocol/tests/pane_reconcile.rs +++ b/crates/freshell-protocol/tests/pane_reconcile.rs @@ -81,6 +81,7 @@ fn ready_capabilities_advertise_pane_reconcile_v1_when_negotiated() { pane_reconcile_v1: Some(true), pane_reconcile_fresh_agent_v1: None, terminal_interest_v1: None, + paced_terminal_replay_v1: None, }), runtime_owners: None, }; diff --git a/crates/freshell-protocol/tests/roundtrip.rs b/crates/freshell-protocol/tests/roundtrip.rs index 729eaa971..00a1aab65 100644 --- a/crates/freshell-protocol/tests/roundtrip.rs +++ b/crates/freshell-protocol/tests/roundtrip.rs @@ -193,6 +193,69 @@ fn ready_carries_build_id_and_omits_it_when_absent() { } } +#[test] +fn hello_roundtrips_paced_terminal_replay_v1_opt_in() { + // Workstream 1 (responsive terminal restore) negotiation: the client opt-in + // rides `hello.capabilities.pacedTerminalReplayV1`. Additive optional — a + // negotiating hello round-trips byte-identically... + let wire = r#"{"type":"hello","protocolVersion":10,"token":"t","capabilities":{"terminalOutputBatchV1":true,"pacedTerminalReplayV1":true}}"#; + match client_roundtrip(wire, "hello") { + ClientMessage::Hello(h) => { + assert_eq!( + h.capabilities.and_then(|c| c.paced_terminal_replay_v1), + Some(true), + "the paced-replay opt-in must parse through the typed struct" + ); + } + other => panic!("expected Hello, got {other:?}"), + } + + // ...and a non-negotiating hello never invents the key (the frozen + // client's wire shape is unchanged). + let wire = r#"{"type":"hello","protocolVersion":10,"token":"t","capabilities":{"terminalOutputBatchV1":true}}"#; + match client_roundtrip(wire, "hello") { + ClientMessage::Hello(h) => { + assert_eq!( + h.capabilities.and_then(|c| c.paced_terminal_replay_v1), + None, + "an absent pacedTerminalReplayV1 must stay absent (skip_serializing_if)" + ); + } + other => panic!("expected Hello, got {other:?}"), + } +} + +#[test] +fn ready_roundtrips_paced_terminal_replay_v1_echo_and_omission() { + // The negotiated echo rides `ready.capabilities` — only for a connection + // whose hello opted in. + let wire = r#"{"type":"ready","timestamp":"2026-09-19T00:00:00.000Z","serverInstanceId":"srv-abc","bootId":"boot-1","capabilities":{"pacedTerminalReplayV1":true}}"#; + match server_roundtrip(wire, "ready") { + ServerMessage::Ready(r) => { + assert_eq!( + r.capabilities.and_then(|c| c.paced_terminal_replay_v1), + Some(true), + "the negotiated paced-replay echo must parse through the typed struct" + ); + } + other => panic!("expected Ready, got {other:?}"), + } + + // A non-negotiating capabilities object stays byte-identical to today's + // output — no paced key is invented for the frozen client. + let wire = r#"{"type":"ready","timestamp":"2026-09-19T00:00:00.000Z","serverInstanceId":"srv-abc","bootId":"boot-1","capabilities":{"paneReconcileV1":true}}"#; + match server_roundtrip(wire, "ready") { + ServerMessage::Ready(r) => { + assert_eq!( + r.capabilities.and_then(|c| c.paced_terminal_replay_v1), + None, + "a non-paced negotiation must not invent pacedTerminalReplayV1" + ); + } + other => panic!("expected Ready, got {other:?}"), + } +} + #[test] fn terminal_inventory_and_settings_parse_from_transcript() { let transcript = read_json("port/oracle/fixtures/handshake-transcript.json"); diff --git a/crates/freshell-terminal/src/registry.rs b/crates/freshell-terminal/src/registry.rs index a571eb1de..ee7f2a612 100644 --- a/crates/freshell-terminal/src/registry.rs +++ b/crates/freshell-terminal/src/registry.rs @@ -136,6 +136,14 @@ struct Subscriber { /// this is set AND `attach_request_id` is present (`broker.ts:1315-1343`); otherwise /// the connection receives legacy per-frame `terminal.output` (the T1 default). terminal_output_batch_v1: bool, + /// `hello.capabilities.pacedTerminalReplayV1` for this connection + /// (responsive-terminal-restore Workstream 1): parked on the subscriber + /// exactly like `terminal_output_batch_v1`. The paced replay core + /// (registry pages + coordinator, task 3) consumes it to gate paced + /// restore delivery; this increment establishes only the negotiation + /// rail, so no reader exists yet. + #[allow(dead_code)] // consumed by Workstream 1 paced replay (task 3) + paced_terminal_replay_v1: bool, } /// One retained produced frame plus its persistent barrier classification (the ring's @@ -1514,6 +1522,7 @@ impl TerminalRegistry { attach_request_id: Option, since_seq: i64, terminal_output_batch_v1: bool, + paced_terminal_replay_v1: bool, session_ref: Option, surface_reset: Option, ) -> AttachOutcome { @@ -1539,6 +1548,7 @@ impl TerminalRegistry { attach_request_id, since_seq, terminal_output_batch_v1, + paced_terminal_replay_v1, session_ref, surface_reset, shared, @@ -1563,6 +1573,7 @@ impl TerminalRegistry { attach_request_id: Option, since_seq: i64, terminal_output_batch_v1: bool, + paced_terminal_replay_v1: bool, session_ref: Option, surface_reset: Option, intent: TerminalAttachIntent, @@ -1584,6 +1595,7 @@ impl TerminalRegistry { attach_request_id, since_seq, terminal_output_batch_v1, + paced_terminal_replay_v1, session_ref, surface_reset, Arc::clone(&handle.shared), @@ -1600,6 +1612,7 @@ impl TerminalRegistry { attach_request_id: Option, since_seq: i64, terminal_output_batch_v1: bool, + paced_terminal_replay_v1: bool, session_ref: Option, surface_reset: Option, shared: Arc>, @@ -1636,6 +1649,7 @@ impl TerminalRegistry { sink: Arc::clone(&sink), attach_request_id: attach_request_id.clone(), terminal_output_batch_v1, + paced_terminal_replay_v1, }, ); // Somebody attached => this terminal is wanted. A later socket drop @@ -4184,6 +4198,7 @@ mod tests { Some("legacy".into()), 0, false, + false, None, None, ); @@ -4216,6 +4231,7 @@ mod tests { Some("batch".into()), 0, true, + false, None, None, ); @@ -4265,7 +4281,7 @@ mod tests { reg.feed("T", frame(1, "a\u{1F600}b\r\n", "S")); // a😀b␍␊ let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("m".into()), 0, true, None, None); + let _ = reg.attach("T", 1, sink, Some("m".into()), 0, true, false, None, None); let bs = batches(&seen); assert_eq!(bs.len(), 1); let b = &bs[0]; @@ -4286,7 +4302,17 @@ mod tests { reg.feed("T", frame(3, "three\r\n", "S")); let (sink, seen) = collector(); - let out = reg.attach("T", 1, sink, Some("att-1".into()), 0, false, None, None); + let out = reg.attach( + "T", + 1, + sink, + Some("att-1".into()), + 0, + false, + false, + None, + None, + ); assert!(out.found); // attach.ready first, then the 3 replayed frames. @@ -4318,7 +4344,17 @@ mod tests { reg.insert_headless("T", "S"); let (sink_a, seen_a) = collector(); - let _ = reg.attach("T", 1, sink_a, Some("a".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 1, + sink_a, + Some("a".into()), + 0, + false, + false, + None, + None, + ); reg.feed("T", frame(1, "before\r\n", "S")); assert_eq!(outputs(&seen_a).len(), 1); @@ -4334,7 +4370,17 @@ mod tests { // A fresh attach replays the FULL scrollback (both frames). let (sink_b, seen_b) = collector(); - let _ = reg.attach("T", 2, sink_b, Some("b".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 2, + sink_b, + Some("b".into()), + 0, + false, + false, + None, + None, + ); let replayed = outputs(&seen_b); assert_eq!( replayed.iter().map(|f| f.data.as_str()).collect::>(), @@ -4349,9 +4395,29 @@ mod tests { let (sink_a, seen_a) = collector(); let (sink_b, seen_b) = collector(); - let _ = reg.attach("T", 1, sink_a, Some("aaa".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 1, + sink_a, + Some("aaa".into()), + 0, + false, + false, + None, + None, + ); // Second attach: geometry authority flips to multi_client_unknown. - let _ = reg.attach("T", 2, sink_b, Some("bbb".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 2, + sink_b, + Some("bbb".into()), + 0, + false, + false, + None, + None, + ); let ready_b = attach_ready(&seen_b).unwrap(); assert_eq!( ready_b.geometry_authority, @@ -4375,7 +4441,17 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink_a, seen_a) = collector(); - let _ = reg.attach("T", 1, sink_a, Some("a".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 1, + sink_a, + Some("a".into()), + 0, + false, + false, + None, + None, + ); for i in 1..=5 { reg.feed("T", frame(i, &format!("line-{i}\r\n"), "S")); } @@ -4385,7 +4461,17 @@ mod tests { // with sinceSeq=3. Only frames 4 and 5 are replayed (seqStart > 3). reg.detach("T", 1); let (sink_r, seen_r) = collector(); - let _ = reg.attach("T", 2, sink_r, Some("a2".into()), 3, false, None, None); + let _ = reg.attach( + "T", + 2, + sink_r, + Some("a2".into()), + 3, + false, + false, + None, + None, + ); let ready = attach_ready(&seen_r).unwrap(); assert_eq!(ready.effective_since_seq, Some(3)); assert_eq!(ready.replay_from_seq, 4); @@ -4404,7 +4490,7 @@ mod tests { reg.feed("T", frame(1, "old\r\n", "S")); let (sink, seen) = collector(); - let _ = reg.attach("T", 7, sink, Some("z".into()), 0, false, None, None); + let _ = reg.attach("T", 7, sink, Some("z".into()), 0, false, false, None, None); // A live frame produced AFTER attach must arrive after the replayed one. reg.feed("T", frame(2, "new\r\n", "S")); @@ -4426,7 +4512,7 @@ mod tests { fn attach_to_unknown_terminal_reports_not_found() { let reg = TerminalRegistry::new(); let (sink, seen) = collector(); - let out = reg.attach("nope", 1, sink, None, 0, false, None, None); + let out = reg.attach("nope", 1, sink, None, 0, false, false, None, None); assert!(!out.found); assert!(seen.lock().unwrap().is_empty()); } @@ -4456,7 +4542,7 @@ mod tests { reg.insert_headless("T", "S"); let rev_before = reg.revision(); let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None); + let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); assert!(reg.kill("T")); assert!(!reg.is_running("T"), "killed terminal is removed"); @@ -4484,8 +4570,8 @@ mod tests { reg.insert_headless("T-b", "S2"); let (sink_a, seen_a) = collector(); let (sink_b, seen_b) = collector(); - let _ = reg.attach("T-a", 1, sink_a, None, 0, false, None, None); - let _ = reg.attach("T-b", 2, sink_b, None, 0, false, None, None); + let _ = reg.attach("T-a", 1, sink_a, None, 0, false, false, None, None); + let _ = reg.attach("T-b", 2, sink_b, None, 0, false, false, None, None); let rev_before = reg.revision(); let killed = reg.kill_all(); @@ -4771,7 +4857,7 @@ mod tests { assert!(!dir[0].has_clients); let (sink, _seen) = collector(); - let _ = reg.attach("T", 9, sink, Some("a".into()), 0, false, None, None); + let _ = reg.attach("T", 9, sink, Some("a".into()), 0, false, false, None, None); assert!(reg.directory()[0].has_clients); reg.detach("T", 9); assert!(!reg.directory()[0].has_clients); @@ -4889,6 +4975,7 @@ mod tests { Some("a-1".into()), 0, false, + false, None, None, TerminalAttachIntent::ViewportHydrate, @@ -4910,6 +4997,7 @@ mod tests { Some("b-1".into()), 0, false, + false, None, None, TerminalAttachIntent::TransportReconnect, @@ -4954,7 +5042,17 @@ mod tests { assert_eq!(out, AttachResizeStatus::Resized); assert_eq!(reg.geometry("T"), Some((131, 48, 1))); let (sink_a, _seen_a) = collector(); - let _ = reg.attach("T", 1, sink_a, Some("a-1".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 1, + sink_a, + Some("a-1".into()), + 0, + false, + false, + None, + None, + ); // A secondary viewer must be able to attach without silently taking // over the shared terminal's geometry. @@ -4962,7 +5060,17 @@ mod tests { assert_eq!(out, AttachResizeStatus::Skipped); assert_eq!(reg.geometry("T"), Some((131, 48, 1))); let (sink_b, _seen_b) = collector(); - let _ = reg.attach("T", 2, sink_b, Some("b-1".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 2, + sink_b, + Some("b-1".into()), + 0, + false, + false, + None, + None, + ); // Later attach generations from that same second socket are still // replay operations, not implicit geometry transfers. @@ -5025,8 +5133,8 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink, _seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None); // conn 1 is attached - // conn 2 reconnects with another socket attached and no prior attachment of its own. + let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); // conn 1 is attached + // conn 2 reconnects with another socket attached and no prior attachment of its own. let out = reg.resize_for_attach("T", 2, TerminalAttachIntent::TransportReconnect, 95, 41); assert_eq!(out, AttachResizeStatus::Skipped); assert_eq!(reg.geometry("T"), Some((120, 30, 1))); @@ -5037,7 +5145,17 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink_a, _seen_a) = collector(); - let _ = reg.attach("T", 1, sink_a, Some("a".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 1, + sink_a, + Some("a".into()), + 0, + false, + false, + None, + None, + ); // The first transport reconnect from B is replay-only while A views // the terminal. Register B so its second generation exercises the @@ -5045,7 +5163,17 @@ mod tests { let out = reg.resize_for_attach("T", 2, TerminalAttachIntent::TransportReconnect, 95, 41); assert_eq!(out, AttachResizeStatus::Skipped); let (sink_b, _seen_b) = collector(); - let _ = reg.attach("T", 2, sink_b, Some("b".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 2, + sink_b, + Some("b".into()), + 0, + false, + false, + None, + None, + ); let out = reg.resize_for_attach("T", 2, TerminalAttachIntent::TransportReconnect, 95, 41); assert_eq!(out, AttachResizeStatus::Skipped); @@ -5084,8 +5212,28 @@ mod tests { reg.insert_headless("T2", "S2"); let (sink1, seen1) = collector(); let (sink2, seen2) = collector(); - let _ = reg.attach("T1", 42, sink1, Some("a".into()), 0, false, None, None); - let _ = reg.attach("T2", 42, sink2, Some("a".into()), 0, false, None, None); + let _ = reg.attach( + "T1", + 42, + sink1, + Some("a".into()), + 0, + false, + false, + None, + None, + ); + let _ = reg.attach( + "T2", + 42, + sink2, + Some("a".into()), + 0, + false, + false, + None, + None, + ); reg.remove_connection(42); // Both terminals survive; the swept connection receives no further output. @@ -5111,7 +5259,7 @@ mod tests { assert!(reg.finish_pty_exit("T", 7)); let (sink, seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None); + let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); assert!(outcome.found); let exit = seen.lock().unwrap().iter().find_map(|m| match m { @@ -5162,6 +5310,7 @@ mod tests { Some("att-1".into()), 0, false, + false, None, Some(true), ); @@ -5204,7 +5353,17 @@ mod tests { reg.feed("T", frame(2, "banner\r\n", "S")); let (sink, seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("b1".into()), 0, true, None, Some(true)); + let outcome = reg.attach( + "T", + 1, + sink, + Some("b1".into()), + 0, + true, + false, + None, + Some(true), + ); assert!(outcome.found); let msgs = seen.lock().unwrap().clone(); @@ -5234,7 +5393,17 @@ mod tests { // Flag absent (None) … let (sink_a, seen_a) = collector(); - let _ = reg.attach("T", 1, sink_a, Some("a".into()), 0, false, None, None); + let _ = reg.attach( + "T", + 1, + sink_a, + Some("a".into()), + 0, + false, + false, + None, + None, + ); // … and explicitly false: no sync either way (fixture f14's gating). let (sink_b, seen_b) = collector(); let _ = reg.attach( @@ -5244,6 +5413,7 @@ mod tests { Some("b".into()), 0, false, + false, None, Some(false), ); @@ -5261,7 +5431,17 @@ mod tests { reg.feed("T", frame(1, "just plain text\r\n", "S")); let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, Some(true)); + let _ = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + Some(true), + ); assert!(modes_syncs(&seen).is_empty(), "empty synthesis => no sync"); } @@ -5274,7 +5454,7 @@ mod tests { // The client fails closed on a sync lacking attachRequestId // (`missing_attach_request_id`), so the server never builds one. let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, None, 0, false, None, Some(true)); + let _ = reg.attach("T", 1, sink, None, 0, false, false, None, Some(true)); assert!( modes_syncs(&seen).is_empty(), "no attachRequestId => no sync" @@ -5292,7 +5472,17 @@ mod tests { assert!(reg.finish_pty_exit("T", 3)); let (sink, seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, Some(true)); + let outcome = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + Some(true), + ); assert!(outcome.found); let msgs = seen.lock().unwrap().clone(); @@ -5367,7 +5557,7 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink, _seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None); + let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); assert!(outcome.found); reg.set_auto_kill_idle_minutes(1); // Far past any threshold, but a client is attached -- legacy: @@ -5544,7 +5734,7 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink, _seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None); + let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); assert!(outcome.found); reg.set_auto_kill_idle_minutes(1); reg.backdate_last_activity("T", now_ms() - 10 * 60_000); @@ -5571,7 +5761,7 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink, _seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None); + let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); assert!(outcome.found); // A second, already-detached terminal whose countdown must NOT be // disturbed by conn 1's disconnect. @@ -5602,7 +5792,7 @@ mod tests { reg.insert_headless("T", "S"); let (sink, _seen) = collector(); assert!( - reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None) + reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None) .found ); reg.set_auto_kill_idle_minutes(15); // the shipped default @@ -5628,7 +5818,7 @@ mod tests { reg.insert_headless("T", "S"); let (sink, _seen) = collector(); assert!( - reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None) + reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None) .found ); reg.set_auto_kill_idle_minutes(15); @@ -5648,13 +5838,13 @@ mod tests { reg.insert_headless("T", "S"); let (sink, _seen) = collector(); assert!( - reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None) + reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None) .found ); reg.detach("T", 1); // explicitly released — fast-reap eligible let (sink2, _seen2) = collector(); assert!( - reg.attach("T", 2, sink2, Some("b".into()), 0, false, None, None) + reg.attach("T", 2, sink2, Some("b".into()), 0, false, false, None, None) .found ); reg.set_auto_kill_idle_minutes(15); @@ -5746,7 +5936,7 @@ mod tests { reg.feed("T", frame(2, "abcdefghij", "S")); // another 10 bytes -> over cap let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None); + let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); let replayed = outputs(&seen); // Whole-frame FIFO eviction keeps at least one frame; the FIRST frame // must have been evicted once the second pushed bytes over the cap. @@ -5764,7 +5954,7 @@ mod tests { reg.feed("T", frame(2, "abcdefghij", "S")); let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, None, None); + let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); let replayed = outputs(&seen); assert_eq!( replayed.len(), @@ -5796,7 +5986,17 @@ mod tests { reg_ascii.feed("A", frame(1, "abcdef", "S")); // 6 chars, 6 bytes reg_ascii.feed("A", frame(2, "ghijkl", "S")); // 6 chars, 6 bytes -> 12 total, at cap let (sink_a, seen_a) = collector(); - let _ = reg_ascii.attach("A", 1, sink_a, Some("r".into()), 0, false, None, None); + let _ = reg_ascii.attach( + "A", + 1, + sink_a, + Some("r".into()), + 0, + false, + false, + None, + None, + ); let ascii_chars: usize = outputs(&seen_a) .iter() .map(|f| f.data.chars().count()) @@ -5815,7 +6015,17 @@ mod tests { frame(2, "\u{2500}\u{2500}\u{2500}\u{2500}\u{2500}\u{2500}", "S"), ); let (sink_b, seen_b) = collector(); - let _ = reg_box.attach("B", 1, sink_b, Some("r".into()), 0, false, None, None); + let _ = reg_box.attach( + "B", + 1, + sink_b, + Some("r".into()), + 0, + false, + false, + None, + None, + ); let box_chars: usize = outputs(&seen_b) .iter() .map(|f| f.data.chars().count()) @@ -6075,7 +6285,7 @@ mod tests { } let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("r".into()), 0, false, None, None); + let _ = reg.attach("T", 1, sink, Some("r".into()), 0, false, false, None, None); let retained_chars: usize = outputs(&seen).iter().map(|f| f.data.chars().count()).sum(); assert!( retained_chars as i64 <= cap, diff --git a/crates/freshell-ws/src/lib.rs b/crates/freshell-ws/src/lib.rs index 43b9cf30e..f5ab1fa6e 100644 --- a/crates/freshell-ws/src/lib.rs +++ b/crates/freshell-ws/src/lib.rs @@ -543,7 +543,7 @@ pub fn spawn_idle_monitor( /// would lose scrollback). On a truly fresh boot the registry is empty, so this stays /// byte-identical to the clean-boot handshake the oracle's T0/determinism tiers pin. pub async fn build_handshake(state: &WsState) -> Vec { - build_handshake_with_capabilities(state, false, false, false).await + build_handshake_with_capabilities(state, false, false, false, false).await } /// [`build_handshake`], parameterized on the connection's negotiated @@ -566,6 +566,7 @@ pub async fn build_handshake_with_capabilities( pane_reconcile_v1: bool, pane_reconcile_fresh_agent_v1: bool, terminal_interest_v1: bool, + paced_terminal_replay_v1: bool, ) -> Vec { let boot_id = state.boot_id.as_ref().clone(); // kata b8ke Task 4 (reconnect-owner discovery, T1 rec A3): replay current @@ -619,11 +620,13 @@ pub async fn build_handshake_with_capabilities( build_id: ready_build_id(), capabilities: (pane_reconcile_v1 || pane_reconcile_fresh_agent_v1 - || terminal_interest_v1) + || terminal_interest_v1 + || paced_terminal_replay_v1) .then_some(freshell_protocol::ReadyCapabilities { pane_reconcile_v1: pane_reconcile_v1.then_some(true), pane_reconcile_fresh_agent_v1: pane_reconcile_fresh_agent_v1.then_some(true), terminal_interest_v1: terminal_interest_v1.then_some(true), + paced_terminal_replay_v1: paced_terminal_replay_v1.then_some(true), }), }), ServerMessage::SettingsUpdated(SettingsUpdated { @@ -843,6 +846,16 @@ async fn handle_socket( .and_then(serde_json::Value::as_bool) .unwrap_or(false); + // Responsive-terminal-restore Workstream 1 (paced replay negotiation): + // same opt-in gate as `paneReconcileV1` — the `ready` echo appears only + // when the client's `hello` opted in, so a frozen client's handshake + // stays byte-for-byte unchanged. + let paced_terminal_replay_v1 = value + .get("capabilities") + .and_then(|c| c.get("pacedTerminalReplayV1")) + .and_then(|v| v.as_bool()) + .unwrap_or(false); + // Authenticated: emit the ordered handshake. CFG-12: the builder is // async + per-connection so its `settings.updated` frame resolves the // LIVE settings tree (see `build_handshake_with_capabilities`). @@ -851,6 +864,7 @@ async fn handle_socket( pane_reconcile_v1, pane_reconcile_fresh_agent_v1, terminal_interest_v1, + paced_terminal_replay_v1, ) .await { @@ -906,6 +920,7 @@ async fn handle_socket( &state, bcast_rx, terminal_output_batch_v1, + paced_terminal_replay_v1, ui_screenshot_v1, pane_reconcile_v1, pane_reconcile_fresh_agent_v1, @@ -1095,7 +1110,7 @@ mod tests { #[tokio::test] async fn handshake_advertises_pane_reconcile_only_when_negotiated() { let s = state(); - let negotiated = build_handshake_with_capabilities(&s, true, false, false).await; + let negotiated = build_handshake_with_capabilities(&s, true, false, false, false).await; let ready = serde_json::to_value(&negotiated[0]).unwrap(); assert_eq!( ready["capabilities"], @@ -1109,11 +1124,44 @@ mod tests { "non-negotiating hello must not change ready's shape: {ready}" ); // Same shape as an explicit `false` negotiation. - let unnegotiated = build_handshake_with_capabilities(&s, false, false, false).await; + let unnegotiated = build_handshake_with_capabilities(&s, false, false, false, false).await; let ready2 = serde_json::to_value(&unnegotiated[0]).unwrap(); assert!(ready2.get("capabilities").is_none()); } + /// Responsive-terminal-restore Workstream 1 (paced replay negotiation): + /// `ready.capabilities.pacedTerminalReplayV1` is advertised ONLY for a + /// hello that opted in — a non-negotiating client's handshake stays + /// byte-identical to the pre-capability shape (frozen-client inertness). + #[tokio::test] + async fn handshake_advertises_paced_terminal_replay_only_when_negotiated() { + let s = state(); + let negotiated = build_handshake_with_capabilities(&s, false, false, false, true).await; + let ready = serde_json::to_value(&negotiated[0]).unwrap(); + assert_eq!( + ready["capabilities"], + serde_json::json!({ "pacedTerminalReplayV1": true }) + ); + + // Non-paced negotiations keep the capabilities object byte-identical + // to today's output — no paced key is invented. + let pane_only = build_handshake_with_capabilities(&s, true, false, false, false).await; + let ready = serde_json::to_value(&pane_only[0]).unwrap(); + assert_eq!( + ready["capabilities"], + serde_json::json!({ "paneReconcileV1": true }), + "a non-paced negotiation must not invent pacedTerminalReplayV1: {ready}" + ); + + // No negotiation at all: no capabilities object on the wire. + let default = build_handshake(&s).await; + let ready = serde_json::to_value(&default[0]).unwrap(); + assert!( + ready.get("capabilities").is_none(), + "non-negotiating hello must not change ready's shape: {ready}" + ); + } + #[tokio::test] async fn handshake_is_ordered_with_shared_bootid() { let msgs = build_handshake(&state()).await; diff --git a/crates/freshell-ws/src/terminal.rs b/crates/freshell-ws/src/terminal.rs index 6b84aaac0..093bc0dc9 100644 --- a/crates/freshell-ws/src/terminal.rs +++ b/crates/freshell-ws/src/terminal.rs @@ -279,6 +279,7 @@ pub async fn run( state: &WsState, bcast_rx: tokio::sync::broadcast::Receiver, terminal_output_batch_v1: bool, + paced_terminal_replay_v1: bool, ui_screenshot_v1: bool, pane_reconcile_v1: bool, pane_reconcile_fresh_agent_v1: bool, @@ -319,6 +320,7 @@ pub async fn run( state, bcast_rx, terminal_output_batch_v1, + paced_terminal_replay_v1, ui_screenshot_v1, pane_reconcile_v1, pane_reconcile_fresh_agent_v1, @@ -341,6 +343,7 @@ async fn run_loop( state: &WsState, mut bcast_rx: tokio::sync::broadcast::Receiver, terminal_output_batch_v1: bool, + paced_terminal_replay_v1: bool, ui_screenshot_v1: bool, pane_reconcile_v1: bool, pane_reconcile_fresh_agent_v1: bool, @@ -506,6 +509,7 @@ async fn run_loop( conn_id, &conn_sink, terminal_output_batch_v1, + paced_terminal_replay_v1, pane_reconcile_v1, pane_reconcile_fresh_agent_v1, &interactive_create_tx, @@ -750,6 +754,7 @@ async fn handle_client_text( conn_id: u64, conn_sink: &FrameSink, terminal_output_batch_v1: bool, + paced_terminal_replay_v1: bool, pane_reconcile_v1: bool, pane_reconcile_fresh_agent_v1: bool, interactive_create_tx: &mpsc::Sender, @@ -1464,6 +1469,7 @@ async fn handle_client_text( conn_id, conn_sink, terminal_output_batch_v1, + paced_terminal_replay_v1, ) { Some(err) => send(ws_tx, &err).await, None => true, @@ -6919,6 +6925,7 @@ fn handle_attach( conn_id: u64, conn_sink: &FrameSink, terminal_output_batch_v1: bool, + paced_terminal_replay_v1: bool, ) -> Option { // STATE-SYNC FIX 1 increment 2a: stamp the canonical identity onto // `attach.ready` from the shared identity registry (create-time @@ -6939,6 +6946,10 @@ fn handle_attach( let outcome = if geometry_identity_ok { let cols = attach.cols.clamp(0, u16::MAX as i64) as u16; let rows = attach.rows.clamp(0, u16::MAX as i64) as u16; + // `paced_terminal_replay_v1` parks the negotiated capability on the + // attach's subscriber (alongside `terminal_output_batch_v1`); + // Workstream 1's paced replay core (registry pages + coordinator, + // task 3) consumes it to gate paced restore delivery. state.registry.attach_with_geometry( &attach.terminal_id, conn_id, @@ -6946,6 +6957,7 @@ fn handle_attach( attach.attach_request_id.clone(), attach.since_seq.unwrap_or(0), terminal_output_batch_v1, + paced_terminal_replay_v1, canonical_session_ref, // Mode replay-sync: the client's positive surface-fresh marker // (xterm recreation / user reset). Forwards the wire field 1:1; the @@ -6963,6 +6975,7 @@ fn handle_attach( attach.attach_request_id.clone(), attach.since_seq.unwrap_or(0), terminal_output_batch_v1, + paced_terminal_replay_v1, canonical_session_ref, attach.surface_reset, ) @@ -10323,6 +10336,7 @@ mod pane_reconcile_gate_tests { 1, &conn_sink, false, + false, false, // pane_reconcile_v1: NOT negotiated on this connection false, &interactive_create_tx, @@ -10349,6 +10363,7 @@ mod pane_reconcile_gate_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, @@ -10393,6 +10408,7 @@ mod pane_reconcile_gate_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, @@ -10417,6 +10433,7 @@ mod pane_reconcile_gate_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, @@ -10758,6 +10775,7 @@ mod host_stats_dispatch_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, @@ -10789,6 +10807,7 @@ mod host_stats_dispatch_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, @@ -10810,6 +10829,7 @@ mod host_stats_dispatch_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, @@ -10852,6 +10872,7 @@ mod host_stats_dispatch_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, @@ -10879,6 +10900,7 @@ mod host_stats_dispatch_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, @@ -10924,6 +10946,7 @@ mod host_stats_dispatch_tests { false, false, false, + false, &interactive_create_tx, &create_cancel_rx, &mut host_stats_last_refresh_at, diff --git a/crates/freshell-ws/tests/hello_capabilities.rs b/crates/freshell-ws/tests/hello_capabilities.rs new file mode 100644 index 000000000..6b42f565f --- /dev/null +++ b/crates/freshell-ws/tests/hello_capabilities.rs @@ -0,0 +1,208 @@ +//! End-to-end capability-negotiation tests for the `/ws` hello→ready handshake +//! (responsive-terminal-restore Workstream 1: `pacedTerminalReplayV1`). +//! +//! These run a REAL axum server on an ephemeral loopback port (never a fixed/ +//! reserved one) and a REAL tokio-tungstenite WS client, so they exercise the +//! actual `handle_socket` path: the raw-JSON `hello` capability extraction and +//! the `ready` advertisement gate — the layers an in-file +//! `build_handshake_with_capabilities` unit test cannot reach (a typo'd wire +//! key in the extraction compiles fine and silently disables negotiation). +//! +//! Backwards-compat contract under test: a hello WITHOUT the capability must +//! produce a `ready` byte-identical to today's output (no `pacedTerminalReplayV1` +//! key, no new capabilities object), on both sides of the change. + +use std::sync::Arc; +use std::time::Duration; + +use futures_util::{SinkExt, StreamExt}; +use tokio::net::TcpListener; +use tokio_tungstenite::tungstenite::Message as WsMessage; + +use freshell_ws::WsState; + +const AUTH_TOKEN: &str = "s3cr3t-token-abcdef"; + +fn test_settings_value() -> serde_json::Value { + serde_json::json!({ + "ai": {}, + "codingCli": { "enabledProviders": [], "mcpServer": true, "providers": {} }, + "editor": { "externalEditor": "auto" }, + "extensions": { "disabled": [] }, + "freshAgent": { "defaultPlugins": [], "enabled": false, "providers": {} }, + "logging": { "debug": false }, + "network": { "configured": true, "host": "127.0.0.1" }, + "panes": { "defaultNewPane": "ask" }, + "safety": { "autoKillIdleMinutes": 15 }, + "sidebar": { + "autoGenerateTitles": true, + "excludeFirstChatMustStart": false, + "excludeFirstChatSubstrings": [] + }, + "terminal": { "scrollback": 10000 } + }) +} + +/// Build a `WsState`, spin up a real axum server on an ephemeral loopback port +/// (`127.0.0.1:0`, never a fixed/reserved port), and return its `ws://` URL. +async fn spawn_server() -> String { + let auth_token = Arc::new(AUTH_TOKEN.to_string()); + let broadcast_tx = Arc::new(tokio::sync::broadcast::channel::(16).0); + let settings = + Arc::new(serde_json::from_value(test_settings_value()).expect("valid settings fixture")); + + let state = WsState { + pane_ledger: std::sync::Arc::new(freshell_ws::pane_ledger::PaneLedger::disabled()), + layout: Default::default(), + identity: freshell_ws::identity::TerminalIdentityRegistry::new(), + terminal_meta: Default::default(), + auth_token: Arc::clone(&auth_token), + server_instance_id: Arc::new("srv-test".to_string()), + boot_id: Arc::new("boot-test".to_string()), + settings, + handshake_settings: Arc::new(tokio::sync::RwLock::new( + serde_json::from_value(test_settings_value()).expect("valid settings fixture"), + )), + broadcast_tx: Arc::clone(&broadcast_tx), + auto_resume_tx: tokio::sync::mpsc::unbounded_channel().0, + auto_resume_cancels: Default::default(), + fresh_codex: freshell_freshagent::FreshCodexState::new( + Arc::clone(&auth_token), + Arc::clone(&broadcast_tx), + serde_json::json!({ "freshAgent": { "enabled": false } }), + ), + fresh_claude: freshell_freshagent::FreshClaudeState::new(Arc::clone(&broadcast_tx)), + fresh_opencode: freshell_freshagent::FreshOpencodeState::new( + freshell_freshagent::FreshAgentState::new( + Arc::clone(&auth_token), + Arc::clone(&broadcast_tx), + ), + ), + registry: freshell_terminal::TerminalRegistry::new(), + tabs: freshell_ws::tabs::TabsRegistry::new(), + screenshots: freshell_ws::screenshot::ScreenshotBroker::new(Arc::clone(&broadcast_tx)), + subagent_interest: Default::default(), + host_stats: Default::default(), + terminals_revision: Arc::new(std::sync::atomic::AtomicI64::new(0)), + sessions_revision: Arc::new(std::sync::atomic::AtomicI64::new(0)), + cli_commands: Arc::new(Vec::new()), + shutdown: Arc::new(tokio::sync::Notify::new()), + ping_interval_ms: 30_000, + hello_timeout_ms: 5_000, + allowed_origins: Arc::new(freshell_ws::origin::default_allowed_origins()), + ws_max_payload_bytes: 16 * 1024 * 1024, + term09: freshell_ws::backpressure::Term09Config::default(), + create_protect: freshell_ws::create_limit::CreateProtectConfig::default(), + spawn_gate: std::sync::Arc::new(freshell_ws::spawn_gate::SpawnGate::new(4, 64)), + shutdown_started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + create_dedupe: std::sync::Arc::new(freshell_ws::create_dedupe::CreateDedupe::default()), + config_fallback: None, + opencode_locator: None, + codex_locator: None, + activity: None, + session_existence: std::sync::Arc::new(freshell_ws::existence::NoIndexProbe::default()), + reconcile_deferral_budget_ms: freshell_ws::reconcile::RECONCILE_DEFERRAL_BUDGET_MS_DEFAULT, + fresh_agent_respawn_counts: Default::default(), + ownership: None, + }; + + let router = freshell_ws::router(state); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("local addr"); + tokio::spawn(async move { + let _ = axum::serve(listener, router).await; + }); + + format!("ws://{addr}/ws", addr = addr) +} + +type WsClient = + tokio_tungstenite::WebSocketStream>; + +/// Send a `hello` and read the `ready` frame back as JSON (the first message +/// of the connect handshake). +async fn hello_ready(ws: &mut WsClient, capabilities: serde_json::Value) -> serde_json::Value { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "hello", + "token": AUTH_TOKEN, + "protocolVersion": freshell_protocol::WS_PROTOCOL_VERSION, + "capabilities": capabilities, + }) + .to_string(), + )) + .await + .expect("send hello"); + + let msg = tokio::time::timeout(Duration::from_secs(5), ws.next()) + .await + .expect("ready within timeout") + .expect("stream not ended") + .expect("no ws error"); + let WsMessage::Text(text) = msg else { + panic!("expected the ready text frame, got {msg:?}"); + }; + serde_json::from_str(&text).expect("ready is JSON") +} + +/// A hello WITH `pacedTerminalReplayV1: true` gets the echo advertised back in +/// `ready.capabilities` — the full negotiation rail (raw-JSON extraction → +/// handshake builder gate) at the real socket. +#[tokio::test] +async fn negotiated_hello_gets_paced_terminal_replay_echo_in_ready() { + let url = spawn_server().await; + let (mut ws, _resp) = tokio_tungstenite::connect_async(&url) + .await + .expect("ws connect"); + + let ready = hello_ready( + &mut ws, + serde_json::json!({ "terminalOutputBatchV1": true, "pacedTerminalReplayV1": true }), + ) + .await; + assert_eq!( + ready["capabilities"], + serde_json::json!({ "pacedTerminalReplayV1": true }), + "a negotiated hello must get exactly the paced-replay echo: {ready}" + ); +} + +/// A hello that negotiates other capabilities but NOT the paced one gets a +/// `ready.capabilities` object byte-identical to today's output — the new key +/// never leaks to a non-opting client. +#[tokio::test] +async fn non_paced_negotiation_keeps_capabilities_byte_identical() { + let url = spawn_server().await; + let (mut ws, _resp) = tokio_tungstenite::connect_async(&url) + .await + .expect("ws connect"); + + let ready = hello_ready( + &mut ws, + serde_json::json!({ "paneReconcileV1": true, "terminalInterestV1": true }), + ) + .await; + assert_eq!( + ready["capabilities"], + serde_json::json!({ "paneReconcileV1": true, "terminalInterestV1": true }), + "a non-paced negotiation must keep today's capabilities shape: {ready}" + ); +} + +/// The frozen client (no capabilities at all) still gets a `ready` with NO +/// capabilities object — byte-identical to the pre-capability handshake. +#[tokio::test] +async fn capability_free_hello_ready_has_no_capabilities_object() { + let url = spawn_server().await; + let (mut ws, _resp) = tokio_tungstenite::connect_async(&url) + .await + .expect("ws connect"); + + let ready = hello_ready(&mut ws, serde_json::json!({})).await; + assert!( + ready.get("capabilities").is_none(), + "a capability-free hello must not change ready's shape: {ready}" + ); +} diff --git a/port/contract/ws-protocol.schema.json b/port/contract/ws-protocol.schema.json index a50fcde4e..2d63d3423 100644 --- a/port/contract/ws-protocol.schema.json +++ b/port/contract/ws-protocol.schema.json @@ -522,6 +522,10 @@ "capabilities": { "additionalProperties": false, "properties": { + "pacedTerminalReplayV1": { + "const": true, + "type": "boolean" + }, "paneReconcileFreshAgentV1": { "const": true, "type": "boolean" @@ -4780,6 +4784,10 @@ "capabilities": { "additionalProperties": false, "properties": { + "pacedTerminalReplayV1": { + "const": true, + "type": "boolean" + }, "paneReconcileFreshAgentV1": { "const": true, "type": "boolean" @@ -7761,6 +7769,10 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "properties": { + "pacedTerminalReplayV1": { + "const": true, + "type": "boolean" + }, "paneReconcileFreshAgentV1": { "const": true, "type": "boolean" diff --git a/port/contract/ws-server-messages.schema.json b/port/contract/ws-server-messages.schema.json index bab92c16f..11524cc6b 100644 --- a/port/contract/ws-server-messages.schema.json +++ b/port/contract/ws-server-messages.schema.json @@ -2656,6 +2656,9 @@ "capabilities": { "additionalProperties": false, "properties": { + "pacedTerminalReplayV1": { + "type": "boolean" + }, "paneReconcileFreshAgentV1": { "type": "boolean" }, diff --git a/shared/ws-protocol.ts b/shared/ws-protocol.ts index 1350dc8d6..a5d3e937b 100644 --- a/shared/ws-protocol.ts +++ b/shared/ws-protocol.ts @@ -419,6 +419,11 @@ export const HelloSchema = z.object({ // STRIP unknown keys, so without this the capability would silently no-op. paneReconcileV1: z.literal(true).optional(), paneReconcileFreshAgentV1: z.literal(true).optional(), + // Paced terminal restore (responsive-terminal-restore Workstream 1): the + // client understands bounded, ascending paced replay batches with + // continuation credit. Additive optional — declared, not just sent (same + // strip hazard as above); absent for the frozen client shape. + pacedTerminalReplayV1: z.literal(true).optional(), }).optional(), client: z.object({ mobile: z.boolean().optional(), @@ -1065,6 +1070,9 @@ export const ReadyCapabilitiesSchema = z terminalInterestV1: z.literal(true).optional(), paneReconcileV1: z.literal(true).optional(), paneReconcileFreshAgentV1: z.literal(true).optional(), + // Paced terminal restore (Workstream 1): echoed only for a hello that + // opted in via capabilities.pacedTerminalReplayV1. + pacedTerminalReplayV1: z.literal(true).optional(), }) .optional() diff --git a/src/lib/ws-client.ts b/src/lib/ws-client.ts index d2d08ff56..e11f22006 100644 --- a/src/lib/ws-client.ts +++ b/src/lib/ws-client.ts @@ -528,7 +528,7 @@ export class WsClient { type: 'hello', token, protocolVersion: WS_PROTOCOL_VERSION, - capabilities: { uiScreenshotV1: true, terminalOutputBatchV1: true, terminalInterestV1: true, paneReconcileV1: true, paneReconcileFreshAgentV1: true }, + capabilities: { uiScreenshotV1: true, terminalOutputBatchV1: true, terminalInterestV1: true, paneReconcileV1: true, paneReconcileFreshAgentV1: true, pacedTerminalReplayV1: true }, ...helloExtensions, }) } diff --git a/test/unit/client/lib/pane-reconcile.test.ts b/test/unit/client/lib/pane-reconcile.test.ts index 954256ed1..a1a28f6c6 100644 --- a/test/unit/client/lib/pane-reconcile.test.ts +++ b/test/unit/client/lib/pane-reconcile.test.ts @@ -393,4 +393,14 @@ describe('schema tests for reconcile v1 widening', () => { // stripped, the feature silently never activates. Assert the key SURVIVES: expect(parsed.success ? parsed.data?.paneReconcileFreshAgentV1 : undefined).toBe(true) }) + + it('ReadyCapabilitiesSchema preserves pacedTerminalReplayV1 through parsing', () => { + // Workstream 1 (responsive terminal restore) negotiation: the paced-replay + // echo must SURVIVE Zod parsing, not just parse successfully — the same + // strip hazard as paneReconcileFreshAgentV1 above (an undeclared key is + // silently stripped and the paced restore feature would never activate). + const parsed = ReadyCapabilitiesSchema.safeParse({ pacedTerminalReplayV1: true }) + expect(parsed.success).toBe(true) + expect(parsed.success ? parsed.data?.pacedTerminalReplayV1 : undefined).toBe(true) + }) }) diff --git a/test/unit/client/lib/ws-client.reconcile.test.ts b/test/unit/client/lib/ws-client.reconcile.test.ts index 173248024..975f69e14 100644 --- a/test/unit/client/lib/ws-client.reconcile.test.ts +++ b/test/unit/client/lib/ws-client.reconcile.test.ts @@ -87,15 +87,35 @@ describe('WsClient pane-reconcile capability', () => { await p }) + it('hello advertises pacedTerminalReplayV1', async () => { + const c = new WsClient('ws://example/ws') + const p = c.connect() + expect(MockWebSocket.instances).toHaveLength(1) + MockWebSocket.instances[0]._open() + + const hello = JSON.parse(MockWebSocket.instances[0].sent[0]) + expect(hello.type).toBe('hello') + expect(hello.capabilities).toMatchObject({ + pacedTerminalReplayV1: true, + }) + + MockWebSocket.instances[0]._message({ type: 'ready' }) + await p + }) + it('surfaces ready.capabilities and resets them on disconnect', async () => { const client = getWsClient() expect(client.getServerCapabilities()).toEqual({}) - await connectAndReady(client, { capabilities: { paneReconcileV1: true } }) + await connectAndReady(client, { + capabilities: { paneReconcileV1: true, pacedTerminalReplayV1: true }, + }) expect(client.getServerCapabilities().paneReconcileV1).toBe(true) + expect(client.getServerCapabilities().pacedTerminalReplayV1).toBe(true) MockWebSocket.instances[0]._close(1006, 'drop') expect(client.getServerCapabilities().paneReconcileV1).toBeUndefined() + expect(client.getServerCapabilities().pacedTerminalReplayV1).toBeUndefined() expect(client.getServerCapabilities()).toEqual({}) }) diff --git a/test/unit/client/lib/ws-client.test.ts b/test/unit/client/lib/ws-client.test.ts index aec503c98..d17cff138 100644 --- a/test/unit/client/lib/ws-client.test.ts +++ b/test/unit/client/lib/ws-client.test.ts @@ -95,6 +95,7 @@ describe('WsClient.connect', () => { terminalInterestV1: true, paneReconcileV1: true, paneReconcileFreshAgentV1: true, + pacedTerminalReplayV1: true, }) MockWebSocket.instances[0]._message({ type: 'ready' }) From d1acb856e29915618e73f03044b9474522040b44 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sat, 19 Sep 2026 23:57:10 -0700 Subject: [PATCH 08/70] feat(protocol): carry restore bounds fields on negotiated attach.ready and output gaps --- .../freshell-protocol/src/server_messages.rs | 39 ++- crates/freshell-protocol/tests/roundtrip.rs | 79 +++++ crates/freshell-terminal/src/lib.rs | 2 +- crates/freshell-terminal/src/registry.rs | 294 +++++++++++++++++- crates/freshell-ws/src/connection_writer.rs | 55 ++++ .../src/connection_writer_tests.rs | 77 +++++ crates/freshell-ws/src/terminal.rs | 14 + .../freshell-ws/tests/hello_capabilities.rs | 125 ++++++++ .../freshell-ws/tests/term09_output_queue.rs | 150 +++++++++ port/contract/ws-server-messages.schema.json | 16 +- shared/ws-protocol.ts | 8 +- 11 files changed, 849 insertions(+), 10 deletions(-) diff --git a/crates/freshell-protocol/src/server_messages.rs b/crates/freshell-protocol/src/server_messages.rs index 8400321dc..60fb536a8 100644 --- a/crates/freshell-protocol/src/server_messages.rs +++ b/crates/freshell-protocol/src/server_messages.rs @@ -339,6 +339,22 @@ pub enum TerminalOutputGapReason { ReplayBudgetExceeded, } +/// `terminal.attach.ready.replayResetReason` — why the attach's effective +/// replay position was reset instead of honoring the requested one +/// (responsive-terminal-restore shared contract). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TerminalReplayResetReason { + /// The geometry-authority check rejected the requested position (the + /// only pre-restore-contract value; const on the wire). + GeometryAuthorityUnknown, + /// The requested position predates the retained replay window + /// (retention loss). Emitted ONLY on connections that negotiated + /// `pacedTerminalReplayV1` — task 3's negotiated retention-gap emission; + /// this increment only extends the value space, no emitter sets it yet. + RetentionLost, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum OutputSource { @@ -1182,9 +1198,16 @@ pub struct TerminalAttachReady { pub geometry_authority: Option, #[serde(skip_serializing_if = "Option::is_none")] pub geometry_epoch: Option, - /// const `"geometry_authority_unknown"`. + /// Restore contract (responsive-terminal-restore): the earliest sequence + /// position still available for replay — the retained ring's front + /// `seqStart`, or `head_seq + 1` when nothing older than the head is + /// retained. Emitted ONLY on connections that negotiated + /// `pacedTerminalReplayV1`; omitted otherwise so the frozen client's + /// frame stays byte-identical. + #[serde(skip_serializing_if = "Option::is_none")] + pub oldest_retained_seq: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub replay_reset_reason: Option, + pub replay_reset_reason: Option, #[serde(skip_serializing_if = "Option::is_none")] pub requested_since_seq: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -1357,6 +1380,18 @@ pub struct TerminalOutputGap { pub to_seq: i64, #[serde(skip_serializing_if = "Option::is_none")] pub attach_request_id: Option, + /// Restore contract (responsive-terminal-restore): the terminal's current + /// `headSeq` at gap-emission time. Emitted ONLY on connections that + /// negotiated `pacedTerminalReplayV1`; omitted otherwise so the frozen + /// client's gap frame stays byte-identical. + #[serde(skip_serializing_if = "Option::is_none")] + pub head_seq: Option, + /// Restore contract (responsive-terminal-restore): the earliest sequence + /// position still available for replay (the retained ring's front + /// `seqStart`, or `head_seq + 1` when the ring is empty) at + /// gap-emission time. Emitted ONLY on negotiated connections. + #[serde(skip_serializing_if = "Option::is_none")] + pub oldest_retained_seq: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/crates/freshell-protocol/tests/roundtrip.rs b/crates/freshell-protocol/tests/roundtrip.rs index 00a1aab65..2a1e1dbe0 100644 --- a/crates/freshell-protocol/tests/roundtrip.rs +++ b/crates/freshell-protocol/tests/roundtrip.rs @@ -654,6 +654,85 @@ fn client_sessions_prefs_roundtrips_and_conforms() { assert_conforms(&validator(&schema), &back, "sessions.prefs"); } +#[test] +fn attach_ready_roundtrips_restore_bounds_and_retention_lost_reset_reason() { + // Responsive-terminal-restore shared contract: negotiated connections get + // the sequence-bounds fields additively — `oldestRetainedSeq` (earliest + // sequence position still available for replay) and the extended + // `replayResetReason` value space. + let wire = r#"{"type":"terminal.attach.ready","terminalId":"t1","streamId":"s1","headSeq":41,"replayFromSeq":7,"replayToSeq":41,"attachRequestId":"a1","requestedSinceSeq":0,"effectiveSinceSeq":0,"oldestRetainedSeq":7}"#; + match server_roundtrip(wire, "terminal.attach.ready") { + ServerMessage::TerminalAttachReady(r) => { + assert_eq!(r.oldest_retained_seq, Some(7)); + assert_eq!(r.replay_reset_reason, None); + } + other => panic!("expected TerminalAttachReady, got {other:?}"), + } + + // The new `retention_lost` reset-reason value round-trips through the + // typed field (task 3 emits it with the negotiated retention gap; this + // contract increment only extends the value space). + let wire = r#"{"type":"terminal.attach.ready","terminalId":"t1","streamId":"s1","headSeq":41,"replayFromSeq":42,"replayToSeq":41,"replayResetReason":"retention_lost","oldestRetainedSeq":42}"#; + match server_roundtrip(wire, "terminal.attach.ready") { + ServerMessage::TerminalAttachReady(r) => { + assert_eq!( + r.replay_reset_reason, + Some(TerminalReplayResetReason::RetentionLost) + ); + assert_eq!(r.oldest_retained_seq, Some(42)); + } + other => panic!("expected TerminalAttachReady, got {other:?}"), + } + + // The pre-existing reset-reason value keeps round-tripping. + let wire = r#"{"type":"terminal.attach.ready","terminalId":"t1","streamId":"s1","headSeq":9,"replayFromSeq":1,"replayToSeq":9,"replayResetReason":"geometry_authority_unknown"}"#; + match server_roundtrip(wire, "terminal.attach.ready") { + ServerMessage::TerminalAttachReady(r) => { + assert_eq!( + r.replay_reset_reason, + Some(TerminalReplayResetReason::GeometryAuthorityUnknown) + ); + } + other => panic!("expected TerminalAttachReady, got {other:?}"), + } + + // The frozen-client shape stays byte-identical: no new keys are invented + // for a connection that did not negotiate the restore contract. + let wire = r#"{"type":"terminal.attach.ready","terminalId":"t1","streamId":"s1","headSeq":3,"replayFromSeq":1,"replayToSeq":3}"#; + match server_roundtrip(wire, "terminal.attach.ready") { + ServerMessage::TerminalAttachReady(r) => { + assert_eq!(r.oldest_retained_seq, None); + assert_eq!(r.replay_reset_reason, None); + } + other => panic!("expected TerminalAttachReady, got {other:?}"), + } +} + +#[test] +fn output_gap_roundtrips_restore_bounds_and_omits_them_for_frozen_clients() { + // Negotiated shape: a queue-overflow gap carries the terminal's current + // `headSeq` and earliest-replayable `oldestRetainedSeq` at emission time. + let wire = r#"{"type":"terminal.output.gap","terminalId":"t1","streamId":"s1","fromSeq":1,"toSeq":9,"reason":"queue_overflow","attachRequestId":"a1","headSeq":12,"oldestRetainedSeq":2}"#; + match server_roundtrip(wire, "terminal.output.gap") { + ServerMessage::TerminalOutputGap(g) => { + assert_eq!(g.head_seq, Some(12)); + assert_eq!(g.oldest_retained_seq, Some(2)); + } + other => panic!("expected TerminalOutputGap, got {other:?}"), + } + + // Non-negotiated shape: both fields stay absent — the frozen client's + // gap frame is byte-identical to the pre-contract wire. + let wire = r#"{"type":"terminal.output.gap","terminalId":"t1","streamId":"s1","fromSeq":1,"toSeq":9,"reason":"queue_overflow"}"#; + match server_roundtrip(wire, "terminal.output.gap") { + ServerMessage::TerminalOutputGap(g) => { + assert_eq!(g.head_seq, None); + assert_eq!(g.oldest_retained_seq, None); + } + other => panic!("expected TerminalOutputGap, got {other:?}"), + } +} + #[test] fn terminal_created_roundtrips_with_and_without_notice() { // Base shape: notice omitted — byte-identical to today's frame on the wire. diff --git a/crates/freshell-terminal/src/lib.rs b/crates/freshell-terminal/src/lib.rs index 9e676a87c..1ec3cbad3 100644 --- a/crates/freshell-terminal/src/lib.rs +++ b/crates/freshell-terminal/src/lib.rs @@ -62,6 +62,6 @@ pub use mode_tracker::ModeTracker; pub use pty::{build_child_env, build_child_env_from_process, MessageSink, PtyTerminal}; pub use registry::{ compute_scrollback_max_bytes, ActivityEvent, ActivityObserver, AttachOutcome, FrameSink, - InputOutcome, TerminalRegistry, + InputOutcome, ReplayBounds, TerminalRegistry, }; pub use replay_ring::{ReplayDeque, ReplayFrame, ReplayRing}; diff --git a/crates/freshell-terminal/src/registry.rs b/crates/freshell-terminal/src/registry.rs index ee7f2a612..0c4163628 100644 --- a/crates/freshell-terminal/src/registry.rs +++ b/crates/freshell-terminal/src/registry.rs @@ -123,6 +123,20 @@ fn now_ms() -> i64 { freshell_platform::clock::now_ms() } +/// Current restore-contract sequence bounds for one terminal's retained +/// replay ring (responsive-terminal-restore shared contract): the +/// terminal's head sequence and the earliest sequence position still +/// available for replay. Payload-free by design — the writer's gap frames +/// stamp these bounds without copying any retained output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplayBounds { + /// Highest produced `seqEnd` (drives `attach.ready.headSeq`). + pub head_seq: i64, + /// The retained ring's front `seqStart`, or `head_seq + 1` when the ring + /// is empty (nothing older than the head is retained). + pub oldest_retained_seq: i64, +} + /// One attached connection's subscription to a terminal's live stream. struct Subscriber { /// Where this connection's frames go (its socket, via a tokio mpsc in `freshell-ws`). @@ -138,10 +152,11 @@ struct Subscriber { terminal_output_batch_v1: bool, /// `hello.capabilities.pacedTerminalReplayV1` for this connection /// (responsive-terminal-restore Workstream 1): parked on the subscriber - /// exactly like `terminal_output_batch_v1`. The paced replay core - /// (registry pages + coordinator, task 3) consumes it to gate paced - /// restore delivery; this increment establishes only the negotiation - /// rail, so no reader exists yet. + /// exactly like `terminal_output_batch_v1`. The shared restore contract's + /// bounds reporting (`attach.ready.oldestRetainedSeq`) is driven by the + /// attach-time parameter in `attach_to_shared`; this per-subscriber copy + /// remains for the paced replay delivery core (registry pages + + /// coordinator, task 3), which has no reader yet. #[allow(dead_code)] // consumed by Workstream 1 paced replay (task 3) paced_terminal_replay_v1: bool, } @@ -293,6 +308,17 @@ struct TerminalShared { } impl TerminalShared { + /// The earliest sequence position still available for replay (restore + /// contract, responsive-terminal-restore): the retained ring's front + /// `seqStart`, or `head_seq + 1` when the ring is empty (nothing older + /// than the head is retained). Caller holds the terminal lock. + fn oldest_retained_seq(&self) -> i64 { + self.replay + .front() + .map(|f| f.output.seq_start) + .unwrap_or(self.head_seq + 1) + } + /// `single_client` while at most one socket is attached; `multi_client_unknown` /// once a second attaches (`§5.3`, `broker.ts:394-395`). The client uses this to /// decide checkpoint/delta-replay validity, so it must reflect reality. @@ -1640,6 +1666,15 @@ impl TerminalRegistry { _ => (head_seq + 1, head_seq), }; + // Restore contract (responsive-terminal-restore): on NEGOTIATED + // attaches only, report the earliest sequence position still + // available for replay — the RING's front (not the attach's replay + // slice), or head+1 when nothing older than the head is retained. + // Captured under the same lock as the replay snapshot so the bound is + // consistent with it. Non-negotiated attaches leave it `None` (the + // frozen client's ready frame stays byte-identical). + let oldest_retained_seq = paced_terminal_replay_v1.then(|| s.oldest_retained_seq()); + // Register BEFORE enqueuing so any live frame the reader appends after we // release the lock is delivered strictly after this replay (the reader is // blocked on this same lock until we return). @@ -1668,6 +1703,7 @@ impl TerminalRegistry { effective_since_seq: Some(effective_since), geometry_authority: Some(s.geometry_authority()), geometry_epoch: Some(s.geometry_epoch), + oldest_retained_seq, replay_reset_reason: None, requested_since_seq: Some(since_seq), session_ref, @@ -1779,6 +1815,31 @@ impl TerminalRegistry { } } + /// Restore-contract sequence bounds for one terminal's retained replay + /// ring (responsive-terminal-restore): current `head_seq` plus the + /// earliest sequence position still available for replay. Takes the + /// per-terminal lock briefly and copies NO payloads. `None` when the + /// terminal does not exist. + /// + /// Callers must NOT hold the calling connection's writer admission lock: + /// the terminal side (subscriber fan-out, attach replay) acquires that + /// lock while holding THIS per-terminal lock, so resolving bounds under + /// the admission lock would invert the established lock order. + pub fn replay_bounds(&self, terminal_id: &str) -> Option { + let shared = { + let inner = self.inner.lock().expect("registry lock"); + inner + .terminals + .get(terminal_id) + .map(|h| Arc::clone(&h.shared)) + }?; + let s = shared.lock().expect("terminal lock"); + Some(ReplayBounds { + head_seq: s.head_seq, + oldest_retained_seq: s.oldest_retained_seq(), + }) + } + /// On socket close: sweep `conn_id` out of EVERY terminal's subscriber set. All /// PTYs keep running (background sessions), reattachable by a future socket. pub fn remove_connection(&self, conn_id: u64) { @@ -4338,6 +4399,231 @@ mod tests { } } + #[test] + fn paced_attach_ready_carries_oldest_retained_seq_from_the_ring_front() { + // Restore contract (responsive-terminal-restore): a NEGOTIATED attach + // (pacedTerminalReplayV1) reports the earliest sequence position still + // available for replay — the retained ring's front seqStart. + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "one\r\n", "S")); + reg.feed("T", frame(2, "two\r\n", "S")); + + let (sink, seen) = collector(); + let _ = reg.attach( + "T", + 1, + sink, + Some("fresh".into()), + 0, + false, + true, + None, + None, + ); + let ready = attach_ready(&seen).expect("attach.ready sent"); + assert_eq!(ready.head_seq, 2); + assert_eq!( + ready.oldest_retained_seq, + Some(1), + "a negotiated fresh attach reports the ring front as the retention bound" + ); + } + + #[test] + fn paced_delta_attach_reports_ring_front_not_the_replay_slice() { + // A delta attach (sinceSeq > 0) replays only the newer frames, but the + // retention bound it reports is the RING's front — the earliest + // position still available for a future re-attach, not this attach's + // replay slice. + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "one\r\n", "S")); + reg.feed("T", frame(2, "two\r\n", "S")); + reg.feed("T", frame(3, "three\r\n", "S")); + + let (sink, seen) = collector(); + let _ = reg.attach( + "T", + 1, + sink, + Some("delta".into()), + 2, + false, + true, + None, + None, + ); + let ready = attach_ready(&seen).expect("attach.ready sent"); + assert_eq!( + ready.replay_from_seq, 3, + "delta attach replays only frame 3" + ); + assert_eq!( + ready.oldest_retained_seq, + Some(1), + "the retention bound is the ring front even when the replay slice starts later" + ); + } + + #[test] + fn paced_attach_ready_on_empty_ring_reports_head_plus_one() { + // Nothing older than the head is retained => the earliest replayable + // position is headSeq+1 (fresh headless terminal: head 0 => 1). + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + + let (sink, seen) = collector(); + let _ = reg.attach( + "T", + 1, + sink, + Some("empty".into()), + 0, + false, + true, + None, + None, + ); + let ready = attach_ready(&seen).expect("attach.ready sent"); + assert_eq!(ready.head_seq, 0); + assert_eq!( + ready.oldest_retained_seq, + Some(1), + "an empty ring retains nothing older than the head: the bound is headSeq+1" + ); + } + + #[test] + fn replay_bounds_reports_head_and_ring_front_without_payloads() { + // The writer-facing accessor: current head plus the earliest + // still-replayable position, in one brief per-terminal lock pass. + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "one\r\n", "S")); + reg.feed("T", frame(2, "two\r\n", "S")); + reg.feed("T", frame(3, "three\r\n", "S")); + assert_eq!( + reg.replay_bounds("T"), + Some(ReplayBounds { + head_seq: 3, + oldest_retained_seq: 1, + }) + ); + } + + #[test] + fn replay_bounds_on_empty_ring_reports_head_plus_one() { + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + assert_eq!( + reg.replay_bounds("T"), + Some(ReplayBounds { + head_seq: 0, + oldest_retained_seq: 1, + }) + ); + } + + #[test] + fn replay_bounds_for_unknown_terminal_is_none() { + let reg = TerminalRegistry::new(); + assert_eq!(reg.replay_bounds("nope"), None); + } + + #[test] + fn unpaced_attach_ready_pins_the_exact_wire_keys_for_both_negotiation_sides() { + // Compatibility invariant (load-bearing): a connection that did NOT + // negotiate sees a ready frame byte-identical to the pre-contract + // shape — `oldestRetainedSeq` is ABSENT from the wire (not null), and + // `replayResetReason` is still absent. The negotiated side adds + // exactly one key: `oldestRetainedSeq`. + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "one\r\n", "S")); + + let (plain_sink, plain_seen) = collector(); + let _ = reg.attach( + "T", + 1, + plain_sink, + Some("plain".into()), + 0, + false, + false, + None, + None, + ); + let plain_ready = attach_ready(&plain_seen).expect("attach.ready sent"); + assert_eq!(plain_ready.oldest_retained_seq, None); + let json = serde_json::to_value(ServerMessage::TerminalAttachReady(plain_ready)).unwrap(); + let mut keys: Vec<&str> = json + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!( + keys, + vec![ + "attachRequestId", + "effectiveSinceSeq", + "geometryAuthority", + "geometryEpoch", + "headSeq", + "replayFromSeq", + "replayToSeq", + "requestedSinceSeq", + "streamId", + "terminalId", + "type", + ], + "the non-negotiated ready frame keeps the pre-contract key set: {json}" + ); + + let (paced_sink, paced_seen) = collector(); + let _ = reg.attach( + "T", + 2, + paced_sink, + Some("paced".into()), + 0, + false, + true, + None, + None, + ); + let paced_ready = attach_ready(&paced_seen).expect("attach.ready sent"); + assert_eq!(paced_ready.oldest_retained_seq, Some(1)); + let json = serde_json::to_value(ServerMessage::TerminalAttachReady(paced_ready)).unwrap(); + let mut keys: Vec<&str> = json + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!( + keys, + vec![ + "attachRequestId", + "effectiveSinceSeq", + "geometryAuthority", + "geometryEpoch", + "headSeq", + "oldestRetainedSeq", + "replayFromSeq", + "replayToSeq", + "requestedSinceSeq", + "streamId", + "terminalId", + "type", + ], + "the negotiated ready frame adds exactly oldestRetainedSeq: {json}" + ); + } + #[test] fn detach_keeps_terminal_running_and_buffering_then_replays_on_reattach() { let reg = TerminalRegistry::new(); diff --git a/crates/freshell-ws/src/connection_writer.rs b/crates/freshell-ws/src/connection_writer.rs index 9a848d48d..3a73bf64f 100644 --- a/crates/freshell-ws/src/connection_writer.rs +++ b/crates/freshell-ws/src/connection_writer.rs @@ -108,12 +108,27 @@ struct Queues { closed: bool, } +/// Emission-time resolver for one terminal's current restore-contract +/// replay-retention bounds (responsive-terminal-restore): installed ONLY on +/// connections whose `hello` negotiated `pacedTerminalReplayV1`, backed by +/// the registry's [`freshell_terminal::TerminalRegistry::replay_bounds`]. +/// Generic over the registry so the writer's unit tests can drive the pump +/// deterministically without a PTY. +type GapBoundsSource = Arc Option + Send + Sync>; + struct Shared { queues: Mutex, output_limit: usize, control_limit: usize, ready: Notify, stop: watch::Sender>, + /// Restore-contract bounds for materialized `terminal.output.gap` + /// frames: set ONLY on negotiated connections, ONCE, by the connection + /// setup BEFORE the pump is spawned (the same pre-spawn setup rule as + /// `enable_terminal_interest`) — a gap can never be leased before the + /// source exists. Unset keeps gap frames byte-identical to the + /// pre-capability wire shape. + gap_bounds: std::sync::OnceLock, } /// A nonblocking, bounded outbox. Its Sink flush means "accepted by this @@ -161,6 +176,7 @@ impl WriterSender { control_limit: control_limit.max(1), ready: Notify::new(), stop: stop_tx, + gap_bounds: std::sync::OnceLock::new(), }); ( Self { @@ -355,6 +371,18 @@ impl WriterSender { .enable(); } + /// Restore contract (responsive-terminal-restore): install the + /// negotiated-connection gap-bounds source. Called ONCE by the + /// connection setup, BEFORE the writer pump is spawned (a gap can never + /// be leased before the source exists). The source resolves a + /// terminal's current `head_seq`/earliest-replayable position at + /// gap-emission time; connections that never negotiated leave the + /// source unset and their gap frames stay byte-identical to the + /// pre-capability wire shape. + pub(super) fn set_paced_replay_gap_bounds(&self, source: GapBoundsSource) { + let _ = self.shared.gap_bounds.set(source); + } + /// Apply one full presentation-interest snapshot. A rejected snapshot is /// returned without replacing the last accepted state; scheduling changes /// are queued-data-only (no attach, resize, spawn, or kill). @@ -488,6 +516,31 @@ impl WriterPump { let (frame, bytes) = match delivery { Delivery::Frame { payload, bytes } => (payload, bytes), Delivery::Gap { terminal_id, range } => { + // Restore contract (responsive-terminal-restore): a + // negotiated connection's gap carries the terminal's + // CURRENT bounds, resolved at emission time. The source + // (registry-backed) takes the per-terminal lock, whose + // holders — subscriber fan-out, attach replays — acquire + // THIS admission lock under theirs, so resolving under + // the admission lock would invert the established lock + // order and can deadlock: release, resolve, re-acquire. + let bounds = match self.shared.gap_bounds.get() { + Some(source) => { + drop(queues); + let bounds = source(&terminal_id); + queues = self.shared.queues.lock().expect("writer queue lock"); + if queues.closed { + // A concurrent stop won the race while the + // admission lock was released. The queue is + // dead (Drop clears it) and the pump returns + // via the stop watch — never lease output + // past a stop. + return Ok(None); + } + bounds + } + None => None, + }; let message = ServerMessage::TerminalOutputGap(freshell_protocol::TerminalOutputGap { terminal_id, @@ -496,6 +549,8 @@ impl WriterPump { from_seq: range.from_seq, to_seq: range.to_seq, reason: freshell_protocol::TerminalOutputGapReason::QueueOverflow, + head_seq: bounds.map(|b| b.head_seq), + oldest_retained_seq: bounds.map(|b| b.oldest_retained_seq), }); let json = serde_json::to_string(&message) .map_err(|_| WriterExit::SerializationFailed)?; diff --git a/crates/freshell-ws/src/connection_writer_tests.rs b/crates/freshell-ws/src/connection_writer_tests.rs index 9db4c7e3b..cbbe6fb69 100644 --- a/crates/freshell-ws/src/connection_writer_tests.rs +++ b/crates/freshell-ws/src/connection_writer_tests.rs @@ -562,3 +562,80 @@ async fn attach_priority_works_for_clients_without_interest_capability() { sender.push_server(named_output("visible", 1)); assert_eq!(taken_terminal(&pump), "visible"); } + +/// Force exactly one queue-overflow eviction: the output limit admits the +/// first frame alone, so the second push evicts the first and materializes +/// its gap. The limit is derived from the probe frame's serialized size so +/// the eviction is deterministic without hardcoding byte counts. +fn overflow_writer() -> (WriterSender, WriterPump) { + let probe = serde_json::to_string(&output(1)).unwrap().len(); + WriterSender::new(probe, 4096, Duration::from_secs(10)) +} + +#[tokio::test] +async fn queue_overflow_gap_carries_restore_bounds_on_paced_connections() { + // Restore contract (responsive-terminal-restore): a connection that + // negotiated pacedTerminalReplayV1 (modeled here by an installed + // gap-bounds source) sees its queue-overflow gaps stamped with the + // terminal's CURRENT headSeq + oldestRetainedSeq, resolved at + // gap-emission (lease) time. + let (sender, pump) = overflow_writer(); + sender.set_paced_replay_gap_bounds(Arc::new(|_| { + Some(freshell_terminal::ReplayBounds { + head_seq: 421, + oldest_retained_seq: 7, + }) + })); + assert!(sender.push_server(output(1))); + assert!(sender.push_server(output(2))); // evicts output(1) -> queue_overflow gap + let next = pump.take_next().unwrap().unwrap(); + let gap: serde_json::Value = serde_json::from_str(&leased_text(&next.frame)).unwrap(); + assert_eq!(gap["type"], "terminal.output.gap"); + assert_eq!(gap["reason"], "queue_overflow"); + assert_eq!(gap["fromSeq"], 1); + assert_eq!(gap["toSeq"], 1); + assert_eq!(gap["headSeq"], 421, "negotiated gap carries headSeq: {gap}"); + assert_eq!( + gap["oldestRetainedSeq"], 7, + "negotiated gap carries oldestRetainedSeq: {gap}" + ); + pump.finish_frame(next.output_bytes, next.control_bytes); + // The evicted frame's successor still delivers. + let next = pump.take_next().unwrap().unwrap(); + assert!(leased_text(&next.frame).contains("data-2")); + pump.finish_frame(next.output_bytes, next.control_bytes); +} + +#[tokio::test] +async fn queue_overflow_gap_without_paced_negotiation_keeps_the_frozen_shape() { + // Compatibility invariant (load-bearing): a connection that did NOT + // negotiate sees gap frames byte-identical to the pre-contract wire — + // no headSeq, no oldestRetainedSeq, and exactly the frozen key set. + let (sender, pump) = overflow_writer(); + assert!(sender.push_server(output(1))); + assert!(sender.push_server(output(2))); // evicts output(1) -> queue_overflow gap + let next = pump.take_next().unwrap().unwrap(); + let gap: serde_json::Value = serde_json::from_str(&leased_text(&next.frame)).unwrap(); + assert_eq!(gap["type"], "terminal.output.gap"); + let mut keys: Vec<&str> = gap + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!( + keys, + vec![ + "attachRequestId", + "fromSeq", + "reason", + "streamId", + "terminalId", + "toSeq", + "type", + ], + "the non-negotiated gap frame keeps the pre-contract key set: {gap}" + ); + pump.finish_frame(next.output_bytes, next.control_bytes); +} diff --git a/crates/freshell-ws/src/terminal.rs b/crates/freshell-ws/src/terminal.rs index 093bc0dc9..fc094bb51 100644 --- a/crates/freshell-ws/src/terminal.rs +++ b/crates/freshell-ws/src/terminal.rs @@ -370,6 +370,20 @@ async fn run_loop( if terminal_interest_v1 { ws_tx.enable_terminal_interest(); } + if paced_terminal_replay_v1 { + // Restore contract (responsive-terminal-restore): a negotiated + // connection's queue-overflow gaps carry the terminal's CURRENT + // replay-retention bounds, resolved from the registry at + // gap-emission time. Installed BEFORE the pump is spawned (the same + // pre-spawn setup rule as `enable_terminal_interest`), so a gap can + // never be leased before the source exists. Non-negotiated + // connections leave the source unset and their gaps stay + // byte-identical to the pre-capability wire shape. + let registry = state.registry.clone(); + ws_tx.set_paced_replay_gap_bounds(Arc::new(move |terminal_id: &str| { + registry.replay_bounds(terminal_id) + })); + } let mut writer_task = tokio::spawn(writer.run(socket_tx).instrument(tracing::Span::current())); let _writer_lifetime = connection_writer::AbortWriterOnDrop(writer_task.abort_handle()); let mut writer_finished = false; diff --git a/crates/freshell-ws/tests/hello_capabilities.rs b/crates/freshell-ws/tests/hello_capabilities.rs index 6b42f565f..c572824bf 100644 --- a/crates/freshell-ws/tests/hello_capabilities.rs +++ b/crates/freshell-ws/tests/hello_capabilities.rs @@ -206,3 +206,128 @@ async fn capability_free_hello_ready_has_no_capabilities_object() { "a capability-free hello must not change ready's shape: {ready}" ); } + +/// Read the next JSON text frame from the socket (bounded). +async fn next_json(ws: &mut WsClient) -> serde_json::Value { + let msg = tokio::time::timeout(Duration::from_secs(5), ws.next()) + .await + .expect("frame within timeout") + .expect("stream not ended") + .expect("no ws error"); + let WsMessage::Text(text) = msg else { + panic!("expected a text frame, got {msg:?}"); + }; + serde_json::from_str(&text).expect("frame is JSON") +} + +/// Send `terminal.create` (shell) and return the created terminalId. +async fn create_shell_terminal(ws: &mut WsClient, request_id: &str) -> String { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.create", + "requestId": request_id, + "mode": "shell", + "shell": "system", + }) + .to_string(), + )) + .await + .expect("send terminal.create"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while tokio::time::Instant::now() < deadline { + let value = next_json(ws).await; + if value.get("type").and_then(|v| v.as_str()) == Some("terminal.created") + && value.get("requestId").and_then(|v| v.as_str()) == Some(request_id) + { + return value + .get("terminalId") + .and_then(|v| v.as_str()) + .expect("terminal.created carries terminalId") + .to_string(); + } + } + panic!("terminal.created never arrived"); +} + +/// Send `terminal.attach` and return the matching `terminal.attach.ready` +/// frame's JSON (skipping any frames in between). +async fn attach_and_read_ready( + ws: &mut WsClient, + terminal_id: &str, + attach_request_id: &str, +) -> serde_json::Value { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.attach", + "terminalId": terminal_id, + "intent": "viewport_hydrate", + "cols": 80, + "rows": 24, + "attachRequestId": attach_request_id, + }) + .to_string(), + )) + .await + .expect("send terminal.attach"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while tokio::time::Instant::now() < deadline { + let value = next_json(ws).await; + if value.get("type").and_then(|v| v.as_str()) == Some("terminal.attach.ready") + && value.get("attachRequestId").and_then(|v| v.as_str()) == Some(attach_request_id) + { + return value; + } + } + panic!("terminal.attach.ready never arrived"); +} + +/// Negotiation gating e2e (restore contract): a `pacedTerminalReplayV1` +/// hello attaching over the REAL socket must see `oldestRetainedSeq` on +/// `terminal.attach.ready` — the full rail (raw hello JSON extraction → +/// `handle_attach` → registry population) that in-file unit tests cannot +/// reach. A fresh terminal reports 1 either way the ring sits (empty ring: +/// headSeq 0 + 1; retained first frame: seqStart 1). +#[tokio::test] +async fn negotiated_attach_ready_carries_oldest_retained_seq() { + let url = spawn_server().await; + let (mut ws, _resp) = tokio_tungstenite::connect_async(&url) + .await + .expect("ws connect"); + let _ready = hello_ready( + &mut ws, + serde_json::json!({ "pacedTerminalReplayV1": true }), + ) + .await; + + let terminal_id = create_shell_terminal(&mut ws, "create-paced").await; + let ready = attach_and_read_ready(&mut ws, &terminal_id, "attach-paced").await; + assert_eq!( + ready["oldestRetainedSeq"], 1, + "a negotiated attach's ready frame must carry the retention bound: {ready}" + ); + assert!( + ready["headSeq"].as_i64().is_some_and(|h| h >= 0), + "headSeq remains the honest current head: {ready}" + ); +} + +/// The compatibility twin: a hello WITHOUT the capability must see a ready +/// frame with NO `oldestRetainedSeq` key at all — byte-identical to the +/// pre-contract shape on the real socket. +#[tokio::test] +async fn plain_attach_ready_omits_oldest_retained_seq() { + let url = spawn_server().await; + let (mut ws, _resp) = tokio_tungstenite::connect_async(&url) + .await + .expect("ws connect"); + let _ready = hello_ready(&mut ws, serde_json::json!({})).await; + + let terminal_id = create_shell_terminal(&mut ws, "create-plain").await; + let ready = attach_and_read_ready(&mut ws, &terminal_id, "attach-plain").await; + assert!( + ready.get("oldestRetainedSeq").is_none(), + "a non-negotiated ready frame must not gain any new key: {ready}" + ); +} diff --git a/crates/freshell-ws/tests/term09_output_queue.rs b/crates/freshell-ws/tests/term09_output_queue.rs index e8a94eb8d..96dc6bc19 100644 --- a/crates/freshell-ws/tests/term09_output_queue.rs +++ b/crates/freshell-ws/tests/term09_output_queue.rs @@ -190,6 +190,31 @@ async fn complete_handshake(ws: &mut TestWs) { } } +/// Same as [`complete_handshake`], but the hello carries a capabilities +/// object (the restore-contract negotiation tests need it). +async fn complete_handshake_with_capabilities(ws: &mut TestWs, capabilities: serde_json::Value) { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "hello", + "token": AUTH_TOKEN, + "protocolVersion": freshell_protocol::WS_PROTOCOL_VERSION, + "capabilities": capabilities, + }) + .to_string(), + )) + .await + .expect("send hello"); + + for _ in 0..4u8 { + let msg = tokio::time::timeout(Duration::from_secs(5), ws.next()) + .await + .expect("handshake message within timeout") + .expect("stream not ended") + .expect("no ws error"); + assert!(matches!(msg, WsMessage::Text(_))); + } +} + async fn create_shell_terminal(ws: &mut TestWs, request_id: &str) -> String { ws.send(WsMessage::Text( serde_json::json!({ @@ -396,3 +421,128 @@ async fn slow_client_does_not_block_fast_client_and_is_bounded() { (catastrophic backpressure fired); observed neither" ); } + +/// Read frames until the first `terminal.output.gap` arrives, returning its +/// JSON. Callers resume a previously-stuck client: the delivery queue serves +/// the terminal's pending gap ahead of its remaining frames, so the gap +/// surfaces within the stuck backlog. +async fn first_gap_frame(ws: &mut TestWs, deadline: tokio::time::Instant) -> serde_json::Value { + while tokio::time::Instant::now() < deadline { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + match tokio::time::timeout(remaining.max(Duration::from_millis(1)), ws.next()).await { + Ok(Some(Ok(WsMessage::Text(text)))) => { + if let Ok(value) = serde_json::from_str::(&text) { + if value.get("type").and_then(|v| v.as_str()) == Some("terminal.output.gap") { + return value; + } + } + } + Ok(Some(Ok(_))) => {} + _ => break, + } + } + panic!("no terminal.output.gap arrived before the deadline"); +} + +/// Restore-contract negotiation gating, end to end on real sockets + a real +/// PTY (responsive-terminal-restore): two slow clients attach to the SAME +/// flooding terminal — one whose hello negotiated `pacedTerminalReplayV1`, +/// one not. The negotiated client's queue-overflow gap carries +/// `headSeq`/`oldestRetainedSeq` resolved from the registry at +/// gap-emission time; the non-negotiated client's gap omits BOTH keys — +/// byte-identical to the pre-contract wire. +#[tokio::test] +async fn queue_overflow_gap_bounds_follow_negotiation() { + // Tiny queue so overflow fires almost immediately; catastrophic + // backpressure threshold far above it so the connection stays OPEN and + // the observed loss is the queue-overflow gap (not a 4008 close). + let term09 = Term09Config { + queue_max_bytes: 8 * 1024, + catastrophic_buffered_bytes: 8 * 1024 * 1024, + catastrophic_stall_ms: 60_000, + }; + let url = spawn_server(term09).await; + + let mut creator = connect_and_complete_handshake(&url).await; + let terminal_id = create_shell_terminal(&mut creator, "create-gap").await; + + // Negotiated slow client: tiny SO_RCVBUF. It reads NOTHING from flood + // start until the fast client below has seen the flood complete — the + // deterministic TERM-09 slow-reader shape (a client that reads along can + // keep the writer draining and no overflow ever fires). + let mut paced = connect_with_tiny_recv_buffer(&url, 4096).await; + complete_handshake_with_capabilities( + &mut paced, + serde_json::json!({ "pacedTerminalReplayV1": true }), + ) + .await; + attach(&mut paced, &terminal_id, "attach-paced").await; + + // Non-negotiated slow client on the SAME terminal, same stuck shape. + let mut plain = connect_with_tiny_recv_buffer(&url, 4096).await; + complete_handshake(&mut plain).await; + attach(&mut plain, &terminal_id, "attach-plain").await; + + // A fast, always-reading client proves when the flood has fully run. + let mut fast = connect_and_complete_handshake(&url).await; + attach(&mut fast, &terminal_id, "attach-fast").await; + + // Let the attach.ready frames settle before flooding. + tokio::time::sleep(Duration::from_millis(200)).await; + + let marker = "FLOOD-DONE-MARKER"; + // ~90 bytes/line * 20_000 lines =~ 1.8 MB: comfortably overruns both + // stuck clients' 8 KB writer queues long before the 60 s send timeout. + let flood = flood_command(20_000, marker); + creator + .send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.input", + "terminalId": terminal_id, + "data": flood, + }) + .to_string(), + )) + .await + .expect("send flood input"); + + // Wait for the flood to COMPLETE on the fast client: by then both stuck + // clients' queues have overflowed and their queue-overflow gaps exist. + let fast_deadline = tokio::time::Instant::now() + Duration::from_secs(20); + let (fast_acc, _fast_gap, fast_closed) = + drain_until_marker_or_deadline(&mut fast, marker, fast_deadline).await; + assert!( + !fast_closed && fast_acc.contains(marker), + "the fast client must see the flood complete before the slow clients resume" + ); + + // NOW resume each stuck client and capture its first queue-overflow gap. + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + let paced_gap = first_gap_frame(&mut paced, deadline).await; + assert_eq!( + paced_gap["reason"], "queue_overflow", + "the negotiated client's observed loss is the queue-overflow gap: {paced_gap}" + ); + let paced_head = paced_gap["headSeq"] + .as_i64() + .expect("negotiated gap carries headSeq"); + let paced_oldest = paced_gap["oldestRetainedSeq"] + .as_i64() + .expect("negotiated gap carries oldestRetainedSeq"); + assert!(paced_head >= 1, "honest current head: {paced_gap}"); + assert!( + paced_oldest >= 1 && paced_oldest <= paced_head + 1, + "oldestRetainedSeq is an honest retention bound (front of the ring, \ + head+1 when empty): {paced_gap}" + ); + + let plain_gap = first_gap_frame(&mut plain, deadline).await; + assert_eq!( + plain_gap["reason"], "queue_overflow", + "the non-negotiated client's observed loss is the queue-overflow gap: {plain_gap}" + ); + assert!( + plain_gap.get("headSeq").is_none() && plain_gap.get("oldestRetainedSeq").is_none(), + "a non-negotiated gap must not gain any new key: {plain_gap}" + ); +} diff --git a/port/contract/ws-server-messages.schema.json b/port/contract/ws-server-messages.schema.json index 11524cc6b..f7945199d 100644 --- a/port/contract/ws-server-messages.schema.json +++ b/port/contract/ws-server-messages.schema.json @@ -3435,11 +3435,17 @@ "headSeq": { "type": "number" }, + "oldestRetainedSeq": { + "type": "number" + }, "replayFromSeq": { "type": "number" }, "replayResetReason": { - "const": "geometry_authority_unknown", + "enum": [ + "geometry_authority_unknown", + "retention_lost" + ], "type": "string" }, "replayToSeq": { @@ -4369,6 +4375,12 @@ "fromSeq": { "type": "number" }, + "headSeq": { + "type": "number" + }, + "oldestRetainedSeq": { + "type": "number" + }, "reason": { "enum": [ "queue_overflow", @@ -4524,9 +4536,9 @@ }, "reason": { "enum": [ + "retention_lost", "new_pty_session", "codex_pty_recovery", - "retention_lost", "server_restart_incompatible_retention" ], "type": "string" diff --git a/shared/ws-protocol.ts b/shared/ws-protocol.ts index a5d3e937b..92f7e5dda 100644 --- a/shared/ws-protocol.ts +++ b/shared/ws-protocol.ts @@ -1240,7 +1240,9 @@ export type TerminalAttachReadyMessage = { geometryAuthority?: TerminalGeometryAuthority requestedSinceSeq?: number effectiveSinceSeq?: number - replayResetReason?: 'geometry_authority_unknown' + /** Restore contract (negotiated pacedTerminalReplayV1 only): earliest sequence position still available for replay (headSeq+1 when nothing older is retained). */ + oldestRetainedSeq?: number + replayResetReason?: 'geometry_authority_unknown' | 'retention_lost' headSeq: number replayFromSeq: number replayToSeq: number @@ -1431,6 +1433,10 @@ export type TerminalOutputGapMessage = { toSeq: number reason: 'queue_overflow' | 'replay_window_exceeded' | 'replay_budget_exceeded' attachRequestId?: string + /** Restore contract (negotiated pacedTerminalReplayV1 only): the terminal's current headSeq at gap-emission time. */ + headSeq?: number + /** Restore contract (negotiated pacedTerminalReplayV1 only): earliest sequence position still available for replay at gap-emission time. */ + oldestRetainedSeq?: number } export type TerminalTitleUpdatedMessage = { From c67de43264fc2775a6243df2b47abd7f381ffbd5 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sun, 20 Sep 2026 02:10:44 -0700 Subject: [PATCH 09/70] feat(ws): paced negotiated terminal replay with credit-gated pages Responsive-terminal-restore Workstream 1 task 3 (the bounded recovery increment's paced path): replace the inline full-replay burst for negotiated connections with bounded, ascending pages driven by a new terminal.replay.credit continuation message. Registry: the negotiated+attachRequestId+Running attach arms the subscriber deferral (the ring becomes the staging), sinks the ready/sync/retention-gap prelude, and returns the FIRST page (selected under the lock, no full-ring clone) plus the session description (fixed target = head at attach, retention-adjusted baseline). next_replay_page packs pages to the serialized budget (batch-builder accounting; an oversized frame forms its own atomic page) and reports Done or the exact Expired interval; next_paced_tail_page drains the accumulated live range and clears the deferral ATOMICALLY when the ring is drained, so live output can never overtake an un-sent page and the flag-clear boundary neither loses nor duplicates a frame. ingest skips deferred subscribers. maxReplayBytes threads through both attach paths and is recorded on the subscriber (TERM-07 seam, no delivery change). WS: the per-connection pacing coordinator (session table, credit window validation, drive loop) produces at most ONE unacknowledged page per credit; a re-attach supersedes the session, detach and socket drop cancel it. Server-pushed terminal.output.gap frames ride the output queue as sequenced controls (never preempting pages). Observability: ws.restore.paced_start/paced_complete/paced_expired/credit events (identifiers and measurements only). Non-negotiated connections stay byte-identical; the frozen legacy pins, batch goldens, and the task-1/2 negotiation suites pass unchanged. Contract artifacts regenerated in lockstep (client surface 41->42, protocol version stays 10). The client's credit-sending behavior is task 4. --- .../freshell-protocol/src/client_messages.rs | 27 +- crates/freshell-protocol/src/lib.rs | 2 +- crates/freshell-protocol/tests/inventory.rs | 12 +- crates/freshell-protocol/tests/roundtrip.rs | 19 + crates/freshell-terminal/src/batch.rs | 35 + crates/freshell-terminal/src/lib.rs | 3 +- crates/freshell-terminal/src/registry.rs | 1929 ++++++++++++++++- crates/freshell-ws/src/connection_writer.rs | 41 +- .../src/connection_writer_tests.rs | 114 + crates/freshell-ws/src/lib.rs | 1 + crates/freshell-ws/src/paced_replay.rs | 475 ++++ crates/freshell-ws/src/terminal.rs | 167 +- crates/freshell-ws/tests/paced_replay.rs | 860 ++++++++ port/contract/ws-message-inventory.json | 3 +- port/contract/ws-protocol.schema.json | 77 +- shared/ws-protocol.ts | 18 + 16 files changed, 3689 insertions(+), 94 deletions(-) create mode 100644 crates/freshell-ws/src/paced_replay.rs create mode 100644 crates/freshell-ws/tests/paced_replay.rs diff --git a/crates/freshell-protocol/src/client_messages.rs b/crates/freshell-protocol/src/client_messages.rs index 109032bce..bddf308ff 100644 --- a/crates/freshell-protocol/src/client_messages.rs +++ b/crates/freshell-protocol/src/client_messages.rs @@ -32,6 +32,15 @@ pub enum ClientMessage { TerminalAttach(TerminalAttach), #[serde(rename = "terminal.interest")] TerminalInterest(TerminalInterest), + /// Responsive-terminal-restore Workstream 1 (paced replay): the + /// continuation credit a `pacedTerminalReplayV1` client sends after fully + /// consuming an ordered replay page — `consumedSeq` is the last sequence + /// it consumed, `attachRequestId` scopes it to one attach generation. + /// Additive optional; protocol version stays 10. Ignored by servers that + /// predate the capability (accept-and-strip) and by connections whose + /// own hello did not negotiate it. + #[serde(rename = "terminal.replay.credit")] + TerminalReplayCredit(TerminalReplayCredit), #[serde(rename = "terminal.autoResumeCancel")] TerminalAutoResumeCancel(TerminalAutoResumeCancel), #[serde(rename = "terminal.detach")] @@ -121,7 +130,7 @@ pub enum ClientMessage { /// The exact `type` discriminants of every client→server message, in the frozen /// inventory's order. This is the T0 conformance checklist. -pub const CLIENT_MESSAGE_TYPES: [&str; 41] = [ +pub const CLIENT_MESSAGE_TYPES: [&str; 42] = [ "amplifier.activity.list", "claude.activity.list", "client.diagnostic", @@ -160,6 +169,7 @@ pub const CLIENT_MESSAGE_TYPES: [&str; 41] = [ "terminal.input", "terminal.interest", "terminal.kill", + "terminal.replay.credit", "terminal.resize", "ui.layout.sync", "ui.screenshot.result", @@ -403,6 +413,21 @@ pub struct TerminalAttach { pub observed_generation: Option, } +/// `terminal.replay.credit` (responsive-terminal-restore Workstream 1): one +/// continuation credit for a paced replay session, granted after the prior +/// page was consumed in order. `consumedSeq` must fall within the server's +/// outstanding-page window `(credited, lastSentPageEnd]`; stale generations +/// (a superseded `attachRequestId`) and out-of-window values are ignored. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TerminalReplayCredit { + pub terminal_id: String, + pub stream_id: String, + pub attach_request_id: String, + /// The last sequence the client fully consumed in order. + pub consumed_seq: i64, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TerminalDetach { diff --git a/crates/freshell-protocol/src/lib.rs b/crates/freshell-protocol/src/lib.rs index b80e0e419..0bb1e0b69 100644 --- a/crates/freshell-protocol/src/lib.rs +++ b/crates/freshell-protocol/src/lib.rs @@ -37,7 +37,7 @@ pub use settings::*; pub const WS_PROTOCOL_VERSION: u32 = 10; /// Every `type` discriminant the protocol speaks, both directions, sorted. -/// (41 client→server + 65 server→client = 106.) +/// (42 client→server + 65 server→client = 107.) pub fn all_message_types() -> Vec<&'static str> { let mut types: Vec<&'static str> = client_messages::CLIENT_MESSAGE_TYPES .iter() diff --git a/crates/freshell-protocol/tests/inventory.rs b/crates/freshell-protocol/tests/inventory.rs index 1b74fbee4..eab534581 100644 --- a/crates/freshell-protocol/tests/inventory.rs +++ b/crates/freshell-protocol/tests/inventory.rs @@ -31,12 +31,12 @@ fn client_types_match_inventory_exactly() { let inv = inventory(); assert_eq!( inv["clientToServer"]["count"].as_u64(), - Some(41), - "inventory declares 41 client→server types" + Some(42), + "inventory declares 42 client→server types" ); let expected = json_type_set(&inv["clientToServer"]["types"]); let actual: BTreeSet = CLIENT_MESSAGE_TYPES.iter().map(|s| s.to_string()).collect(); - assert_eq!(actual.len(), 41, "crate declares 41 client types (no dups)"); + assert_eq!(actual.len(), 42, "crate declares 42 client types (no dups)"); assert_eq!( actual, expected, "CLIENT_MESSAGE_TYPES must equal the frozen inventory (no missing/extra)" @@ -61,14 +61,14 @@ fn server_types_match_inventory_exactly() { } #[test] -fn combined_surface_is_106() { +fn combined_surface_is_107() { let all = all_message_types(); - assert_eq!(all.len(), 106, "41 client + 65 server = 106 discriminants"); + assert_eq!(all.len(), 107, "42 client + 65 server = 107 discriminants"); // sorted + unique let unique: BTreeSet<&str> = all.iter().copied().collect(); assert_eq!( unique.len(), - 106, + 107, "no discriminant collides across directions" ); } diff --git a/crates/freshell-protocol/tests/roundtrip.rs b/crates/freshell-protocol/tests/roundtrip.rs index 2a1e1dbe0..b09549855 100644 --- a/crates/freshell-protocol/tests/roundtrip.rs +++ b/crates/freshell-protocol/tests/roundtrip.rs @@ -733,6 +733,25 @@ fn output_gap_roundtrips_restore_bounds_and_omits_them_for_frozen_clients() { } } +#[test] +fn terminal_replay_credit_roundtrips_and_conforms() { + // Responsive-terminal-restore Workstream 1 (paced replay): the + // continuation-credit message a pacedTerminalReplayV1 client sends after + // consuming an ordered replay page — additive optional, protocol version + // stays 10. Task 3 owns the Rust struct + Zod schema in lockstep; the + // client's sending behavior is task 4. + let wire = r#"{"type":"terminal.replay.credit","terminalId":"t1","streamId":"s1","attachRequestId":"a1","consumedSeq":41}"#; + match client_roundtrip(wire, "terminal.replay.credit") { + ClientMessage::TerminalReplayCredit(credit) => { + assert_eq!(credit.terminal_id, "t1"); + assert_eq!(credit.stream_id, "s1"); + assert_eq!(credit.attach_request_id, "a1"); + assert_eq!(credit.consumed_seq, 41); + } + other => panic!("expected TerminalReplayCredit, got {other:?}"), + } +} + #[test] fn terminal_created_roundtrips_with_and_without_notice() { // Base shape: notice omitted — byte-identical to today's frame on the wire. diff --git a/crates/freshell-terminal/src/batch.rs b/crates/freshell-terminal/src/batch.rs index b5b06c73c..3955be8d0 100644 --- a/crates/freshell-terminal/src/batch.rs +++ b/crates/freshell-terminal/src/batch.rs @@ -133,6 +133,41 @@ fn measure_json_bytes(value: &Value) -> usize { .len() } +/// The FIXED scaffold of the legacy `terminal.output` envelope — every byte +/// of [`measure_legacy_output_bytes`] except the `seqStart`/`seqEnd` digit +/// widths and the JSON-escaped `data` literal — measured once so a paced +/// page walk can account frames INCREMENTALLY (responsive-terminal-restore): +/// +/// `measure(seq_start, seq_end, data) == scaffold + digits(seq_start) +/// + digits(seq_end) + json_escaped_len(data)` +/// +/// exactly, with no per-candidate re-serialization of the accumulated run. +pub(crate) fn legacy_envelope_scaffold_bytes( + terminal_id: &str, + stream_id: &str, + attach_request_id: Option<&str>, + source: Option<&str>, +) -> usize { + // seqs "0"/"0" contribute one digit each; the empty data contributes + // exactly its two quote characters. + measure_legacy_output_bytes(terminal_id, stream_id, 0, 0, "", attach_request_id, source) + .saturating_sub(4) +} + +/// The compact-JSON byte length of a string payload INCLUDING its quotes +/// (exactly the `"data":` segment the envelope measure accounts). +pub(crate) fn json_escaped_len(data: &str) -> usize { + serde_json::to_string(data) + .expect("terminal data is always serializable") + .len() +} + +/// Decimal digit count of an i64 (0 has one digit; negatives never occur for +/// seqs but stay honest). +pub(crate) fn digit_count(n: i64) -> usize { + n.checked_abs().unwrap_or(i64::MAX).to_string().len() +} + /// `defaultPayloadForFrame` (`output-batch.ts:83-99`) measured as the legacy /// `terminal.output` envelope — the merge-budget size for `data`. fn measure_legacy_output_bytes( diff --git a/crates/freshell-terminal/src/lib.rs b/crates/freshell-terminal/src/lib.rs index 1ec3cbad3..eeddc3a6e 100644 --- a/crates/freshell-terminal/src/lib.rs +++ b/crates/freshell-terminal/src/lib.rs @@ -62,6 +62,7 @@ pub use mode_tracker::ModeTracker; pub use pty::{build_child_env, build_child_env_from_process, MessageSink, PtyTerminal}; pub use registry::{ compute_scrollback_max_bytes, ActivityEvent, ActivityObserver, AttachOutcome, FrameSink, - InputOutcome, ReplayBounds, TerminalRegistry, + InputOutcome, PacedAttachStart, PacedPage, PacedSessionDesc, PacedTailPage, ReplayBounds, + TerminalRegistry, DEFAULT_PACED_PAGE_MAX_BYTES, }; pub use replay_ring::{ReplayDeque, ReplayFrame, ReplayRing}; diff --git a/crates/freshell-terminal/src/registry.rs b/crates/freshell-terminal/src/registry.rs index 0c4163628..8be51108d 100644 --- a/crates/freshell-terminal/src/registry.rs +++ b/crates/freshell-terminal/src/registry.rs @@ -51,7 +51,7 @@ use freshell_platform::SpawnSpec; use freshell_protocol::{ GeometryAuthority, InventoryTerminal, OutputSource, ServerMessage, SessionLocator, TerminalAttachIntent, TerminalAttachReady, TerminalExit, TerminalModesSync, TerminalOutput, - TerminalRunStatus, + TerminalOutputGap, TerminalOutputGapReason, TerminalReplayResetReason, TerminalRunStatus, }; use crate::barrier_scanner::{BarrierReason, BarrierScanner, ScannerState}; @@ -81,6 +81,15 @@ const MIN_SCROLLBACK_CHARS: i64 = 64 * 1024; const MAX_SCROLLBACK_CHARS: i64 = 4 * 1024 * 1024; /// `APPROX_CHARS_PER_LINE` (`terminal-registry.ts:60`). const APPROX_CHARS_PER_LINE: i64 = 300; +/// Responsive-terminal-restore Workstream 1: the default serialized-byte +/// budget of ONE paced replay page (the plan's "128 KiB initial terminal +/// batch target" — a per-page delivery bound, NOT a claim about total +/// reconstruction size). The budget covers the JSON envelope, escaping, +/// and batch-segment metadata; a single frame whose own envelope exceeds it +/// forms its own atomic single-frame page (guaranteed progress). Held as a +/// registry-level atomic (like `scrollback_max_bytes`) so focused tests can +/// shrink it per-instance without env races. +pub const DEFAULT_PACED_PAGE_MAX_BYTES: i64 = 128 * 1024; /// `computeScrollbackMaxChars(settings)` (`terminal-registry.ts:1328-1333`): /// `settings.terminal.scrollback` LINES converted to an approximate **CHAR** @@ -137,6 +146,103 @@ pub struct ReplayBounds { pub oldest_retained_seq: i64, } +/// The ws pacing coordinator's session description for one paced replay +/// (responsive-terminal-restore Workstream 1): everything the coordinator +/// needs to gate continuation credits and drive page reads. Produced by the +/// paced attach, owned by the connection (one session per +/// (connection, terminal); a re-attach replaces it). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PacedSessionDesc { + pub terminal_id: String, + pub stream_id: String, + /// The attach generation this session serves; stale-credit rejection key. + pub attach_request_id: String, + /// FIXED catch-up target: `head_seq` at attach time. Ongoing output never + /// extends it — frames past the target are tail-phase delivery. + pub target: i64, + /// The effective replay baseline (the requested `sinceSeq` clamped to + /// ≥ 0, reset to `oldest-1` on attach-time retention loss). + pub effective_since: i64, + /// The production cursor: the last seq sent in a page (== the first + /// page's last seq; `effective_since` when the first page is empty). + pub page_end: i64, + /// Serialized wire bytes of the first page (observability). + pub page_bytes: u64, +} + +/// What a negotiated (paced) attach returns to the ws layer INSTEAD of an +/// inline replay burst: the session description plus the FIRST page's wire +/// messages. The ws layer sinks the page AFTER the terminal lock is +/// released; subsequent pages are produced on continuation credit and the +/// tail range `(target, head]` is drained as ordinary delivery. +#[derive(Debug, Clone, PartialEq)] +pub struct PacedAttachStart { + pub session: PacedSessionDesc, + pub first_page: Vec, +} + +/// Result of [`TerminalRegistry::next_replay_page`] — one bounded, +/// ascending page of the `(from_seq, target]` window, the window's +/// completion, an exact retention-loss report, or the session's +/// disappearance. +#[derive(Debug, Clone, PartialEq)] +#[must_use] +pub enum PacedPage { + /// One packed page: ascending wire messages, the page's last seq (the + /// session's new cursor), and the page's total serialized bytes. + Frames { + messages: Vec, + end_seq: i64, + serialized_bytes: u64, + }, + /// `from_seq >= target` — the replay window is fully delivered. + Done, + /// Retention evicted the frames the session needs next. The exact lost + /// interval is `[lost_from, lost_to]` and the session continues from + /// `resume_from` (the new ring front − 1); `head_seq`/`oldest_retained_seq` + /// are the task-2 bounds fields for the negotiated gap frame. + Expired { + lost_from: i64, + lost_to: i64, + resume_from: i64, + head_seq: i64, + oldest_retained_seq: i64, + }, + /// The terminal (or this connection's subscriber) is gone — cancel the + /// session; there is no deferral left to clear. + Gone, +} + +/// Result of [`TerminalRegistry::next_paced_tail_page`] — the post-replay +/// catch-up phase: pages over `(from_seq, head]` until the ring is drained, +/// at which point the deferral clears ATOMICALLY under the same lock hold +/// and live output resumes direct fan-out. +#[derive(Debug, Clone, PartialEq)] +#[must_use] +pub enum PacedTailPage { + /// One packed page of the accumulated live range. + Frames { + messages: Vec, + end_seq: i64, + serialized_bytes: u64, + }, + /// Retention evicted part of the tail range — the same exact-interval + /// report as [`PacedPage::Expired`]. + Expired { + lost_from: i64, + lost_to: i64, + resume_from: i64, + head_seq: i64, + oldest_retained_seq: i64, + }, + /// The ring is drained and the deferral is CLEARED (under this lock + /// hold): frames appended after this point fan out directly. The paced + /// session is complete. + CaughtUp, + /// The terminal or subscriber is gone — cancel the session. + Gone, +} + /// One attached connection's subscription to a terminal's live stream. struct Subscriber { /// Where this connection's frames go (its socket, via a tokio mpsc in `freshell-ws`). @@ -152,13 +258,30 @@ struct Subscriber { terminal_output_batch_v1: bool, /// `hello.capabilities.pacedTerminalReplayV1` for this connection /// (responsive-terminal-restore Workstream 1): parked on the subscriber - /// exactly like `terminal_output_batch_v1`. The shared restore contract's - /// bounds reporting (`attach.ready.oldestRetainedSeq`) is driven by the - /// attach-time parameter in `attach_to_shared`; this per-subscriber copy - /// remains for the paced replay delivery core (registry pages + - /// coordinator, task 3), which has no reader yet. - #[allow(dead_code)] // consumed by Workstream 1 paced replay (task 3) + /// exactly like `terminal_output_batch_v1`. Gates the registry's paced + /// page reads — a subscriber that did not negotiate never serves them. paced_terminal_replay_v1: bool, + /// Restore contract (responsive-terminal-restore): while a paced replay + /// session is active for this (connection, terminal), [`ingest`] does NOT + /// fan output out to this subscriber — the retained ring IS the staging + /// and the session's pages deliver the range in seq order. Armed by the + /// paced attach, cleared ATOMICALLY under the per-terminal lock by the + /// tail read that finds the ring drained (`next_paced_tail_page`), so + /// the flag-clear + page-read boundary can neither lose nor duplicate a + /// frame: everything appended before the clear is paged, everything + /// appended after it is fanned out directly. + paced_deferred: bool, + /// TERM-07 seam: the attach's `maxReplayBytes` request, threaded through + /// BOTH attach paths and recorded here with NO delivery-behavior change + /// this increment. The plan's binding rule preserves the field's legacy + /// serialized-tail-budget meaning and forbids interpreting it under the + /// paced capability (no newest-tail selection exists until the + /// validated-baseline/screen-snapshot increment); the paced-start + /// observability event reports it (from the wire frame, in the ws + /// layer), and the increment-3 snapshot work consumes this record. + #[allow(dead_code)] + // the increment-3 snapshot work reads it; nothing may read it THIS increment + max_replay_bytes: Option, } /// One retained produced frame plus its persistent barrier classification (the ring's @@ -652,6 +775,10 @@ pub struct TerminalRegistry { /// Captured into each new terminal's `max_replay_chars` at [`Self::create`] /// time (TERM-13) -- see [`compute_scrollback_max_bytes`]. scrollback_max_bytes: Arc, + /// Responsive-terminal-restore Workstream 1: the serialized-byte budget + /// of ONE paced replay page ([`DEFAULT_PACED_PAGE_MAX_BYTES`]). Atomic so + /// focused tests shrink it per-instance without env races. + paced_page_max_bytes: Arc, /// TERM-15/TERM-16 activity tap (see [`ActivityEvent`]). Set once at boot /// by the activity hub; `None` (the default) keeps every fire point a /// cheap no-op. RwLock: read per event, written once. @@ -756,7 +883,7 @@ impl Default for TerminalRegistry { /// `attach.ready` + replay were enqueued to the caller's sink) — `false` draws the /// reference's `INVALID_TERMINAL_ID` reply (attach to an unknown terminal; an /// exited-but-still-registered terminal is `found: true` + a synthetic exit). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] #[must_use] pub struct AttachOutcome { pub found: bool, @@ -764,6 +891,12 @@ pub struct AttachOutcome { /// [`TerminalRegistry::attach_with_geometry`]. Plain [`TerminalRegistry::attach`] /// has no geometry input and returns `None`. pub geometry: Option, + /// A negotiated (pacedTerminalReplayV1 + attachRequestId) attach to a + /// RUNNING terminal: the paced session start — the first page's wire + /// messages plus the session description — INSTEAD of an inline replay + /// burst. `None` on every legacy path (non-negotiated, missing + /// attachRequestId, already-exited terminal). + pub paced: Option, } /// Outcome of [`TerminalRegistry::input`]: whether the terminal existed (the @@ -946,6 +1079,7 @@ impl TerminalRegistry { active_connections: Arc::new(AtomicI64::new(0)), auto_kill_idle_minutes: Arc::new(AtomicI64::new(DEFAULT_AUTO_KILL_IDLE_MINUTES)), scrollback_max_bytes: Arc::new(AtomicI64::new(DEFAULT_MAX_SCROLLBACK_CHARS)), + paced_page_max_bytes: Arc::new(AtomicI64::new(DEFAULT_PACED_PAGE_MAX_BYTES)), activity_observer: Arc::new(std::sync::RwLock::new(None)), respawn_liveness_window_ms: Arc::new(AtomicI64::new( DEFAULT_RESPAWN_LIVENESS_WINDOW_MS, @@ -1170,6 +1304,22 @@ impl TerminalRegistry { self.scrollback_max_bytes.load(Ordering::Relaxed) } + /// Responsive-terminal-restore Workstream 1: update the serialized-byte + /// budget of one paced replay page. Applied to every page produced after + /// the call (the per-page read takes the terminal lock briefly, so a + /// live change is safe). Tests use small values for deterministic + /// multi-page fixtures. + pub fn set_paced_page_max_bytes(&self, max_bytes: i64) { + self.paced_page_max_bytes + .store(max_bytes, Ordering::Relaxed); + } + + /// The current paced-page serialized-byte budget + /// ([`DEFAULT_PACED_PAGE_MAX_BYTES`] unless configured). + pub fn paced_page_max_bytes(&self) -> i64 { + self.paced_page_max_bytes.load(Ordering::Relaxed) + } + /// Reconciliation §7.5: shrink/grow the liveness window a generation must /// survive to reset the respawn counter (tests use small values). pub fn set_respawn_liveness_window_ms(&self, ms: i64) { @@ -1530,7 +1680,7 @@ impl TerminalRegistry { /// `reconcileTerminalSessionAssociation`, a repair channel that was dead /// while this frame hardcoded `None`). /// - /// 9 arguments (`clippy::too_many_arguments`): every one is a distinct, + /// 10 arguments (`clippy::too_many_arguments`): every one is a distinct, /// non-optional attach input with exactly one call site outside tests /// (`freshell_ws::terminal::handle_attach`, which forwards the parsed /// `terminal.attach` frame fields 1:1) — a params struct would just @@ -1539,6 +1689,9 @@ impl TerminalRegistry { /// marker (mode replay-sync); when `Some(true)` and an /// `attach_request_id` is present, the tracker-synthesized mode preamble /// is emitted once, strictly between `attach.ready` and the replay. + /// `max_replay_bytes` is the attach's TERM-07 budget request — recorded + /// on the subscriber with no delivery-behavior change (see + /// [`Subscriber::max_replay_bytes`]). #[allow(clippy::too_many_arguments)] pub fn attach( &self, @@ -1551,6 +1704,7 @@ impl TerminalRegistry { paced_terminal_replay_v1: bool, session_ref: Option, surface_reset: Option, + max_replay_bytes: Option, ) -> AttachOutcome { // Take the terminal's shared Arc under the registry lock, then drop the // registry lock so we hold ONLY the per-terminal lock during the handoff. @@ -1562,6 +1716,7 @@ impl TerminalRegistry { return AttachOutcome { found: false, geometry: None, + paced: None, } } } @@ -1577,6 +1732,7 @@ impl TerminalRegistry { paced_terminal_replay_v1, session_ref, surface_reset, + max_replay_bytes, shared, None, ) @@ -1602,6 +1758,7 @@ impl TerminalRegistry { paced_terminal_replay_v1: bool, session_ref: Option, surface_reset: Option, + max_replay_bytes: Option, intent: TerminalAttachIntent, cols: u16, rows: u16, @@ -1611,6 +1768,7 @@ impl TerminalRegistry { return AttachOutcome { found: false, geometry: Some(AttachResizeStatus::Missing), + paced: None, }; }; @@ -1624,6 +1782,7 @@ impl TerminalRegistry { paced_terminal_replay_v1, session_ref, surface_reset, + max_replay_bytes, Arc::clone(&handle.shared), Some((intent, cols, rows, handle.pty.as_ref())), ) @@ -1641,6 +1800,7 @@ impl TerminalRegistry { paced_terminal_replay_v1: bool, session_ref: Option, surface_reset: Option, + max_replay_bytes: Option, shared: Arc>, geometry: Option<(TerminalAttachIntent, u16, u16, Option<&PtyTerminal>)>, ) -> AttachOutcome { @@ -1648,6 +1808,34 @@ impl TerminalRegistry { let geometry = geometry.map(|(intent, cols, rows, pty)| { apply_attach_geometry(&mut s, intent, cols, rows, pty) }); + + // Responsive-terminal-restore Workstream 1: the paced path replaces + // the inline full-replay burst ONLY for negotiated connections — and + // only for a RUNNING terminal with an attachRequestId to correlate + // continuation credits (the already-Exited path keeps the frozen + // inline replay + synthetic exit, in their legacy order; a paced + // attach without an attachRequestId cannot be credited and falls + // back to the legacy inline replay, byte-identical to today). + let paced = paced_terminal_replay_v1 + && attach_request_id.is_some() + && s.status == TerminalRunStatus::Running; + + if paced { + return self.paced_attach_to_shared( + s, + terminal_id, + conn_id, + sink, + attach_request_id, + since_seq, + terminal_output_batch_v1, + session_ref, + surface_reset, + max_replay_bytes, + geometry, + ); + } + let effective_since = since_seq.max(0); // Snapshot the replay window: every retained frame newer than the client's @@ -1685,6 +1873,8 @@ impl TerminalRegistry { attach_request_id: attach_request_id.clone(), terminal_output_batch_v1, paced_terminal_replay_v1, + paced_deferred: false, + max_replay_bytes, }, ); // Somebody attached => this terminal is wanted. A later socket drop @@ -1780,6 +1970,176 @@ impl TerminalRegistry { AttachOutcome { found: true, geometry, + paced: None, + } + } + + /// The PACED attach handoff (responsive-terminal-restore Workstream 1): + /// the negotiated Running-terminal replacement for the inline + /// full-replay burst. Under the same per-terminal lock as the legacy + /// path: apply geometry, resolve the retention-adjusted baseline, arm + /// the subscriber's deferral (the ring becomes the staging), sink the + /// ready/sync/retention-gap prelude, and SELECT the first page's frames + /// (never a full-ring clone). The first page travels back to the ws + /// caller — it is sunk only after the lock is released; the ws pacing + /// coordinator owns the session (credits, tail drain, completion). + /// + /// Retention loss at attach (the requested baseline predates the + /// retained ring): emit the negotiated `terminal.output.gap` with reason + /// `replay_window_exceeded` for the exact lost interval + /// `[effective+1, oldest-1]` plus the task-2 bounds fields, stamp the + /// ready frame's `replayResetReason: retention_lost`, and CONTINUE from + /// what is retained (baseline `oldest-1`) — nothing is killed, nothing + /// stalls; the client shows the honest incomplete-history state. + /// Non-negotiated attaches keep today's silent behavior exactly (see + /// the legacy branch above). + #[allow(clippy::too_many_arguments)] + fn paced_attach_to_shared( + &self, + mut s: std::sync::MutexGuard<'_, TerminalShared>, + terminal_id: &str, + conn_id: u64, + sink: FrameSink, + attach_request_id: Option, + since_seq: i64, + terminal_output_batch_v1: bool, + session_ref: Option, + surface_reset: Option, + max_replay_bytes: Option, + geometry: Option, + ) -> AttachOutcome { + let effective_requested = since_seq.max(0); + let head_seq = s.head_seq; + let oldest = s.oldest_retained_seq(); + + // Retention loss: the first position the client needs + // (`effective+1`) predates the retained ring. + let retention_lost = effective_requested + 1 < oldest; + let baseline = if retention_lost { + oldest - 1 + } else { + effective_requested + }; + let arid = attach_request_id + .clone() + .expect("the paced path requires an attachRequestId"); + + // Arm the deferral with the subscriber installation: from this + // point until the session completes, ingest does NOT fan out to + // this subscriber — the pages and the tail deliver its range in + // seq order (the ring is the staging). + s.subscribers.insert( + conn_id, + Subscriber { + sink: Arc::clone(&sink), + attach_request_id: Some(arid.clone()), + terminal_output_batch_v1, + paced_terminal_replay_v1: true, + paced_deferred: true, + max_replay_bytes, + }, + ); + s.released_by_client = false; + + // replayFrom/To describe the FULL window the session will deliver + // (the same first/last-span meaning as legacy, projected onto the + // paced range): `(baseline, head]`, or the empty span when the + // baseline already sits at the head. + let (replay_from, replay_to) = if baseline >= head_seq { + (head_seq + 1, head_seq) + } else { + (baseline + 1, head_seq) + }; + + let ready = ServerMessage::TerminalAttachReady(TerminalAttachReady { + head_seq, + replay_from_seq: replay_from, + replay_to_seq: replay_to, + stream_id: s.stream_id.clone(), + terminal_id: terminal_id.to_string(), + attach_request_id: Some(arid.clone()), + effective_since_seq: Some(baseline), + geometry_authority: Some(s.geometry_authority()), + geometry_epoch: Some(s.geometry_epoch), + oldest_retained_seq: Some(oldest), + replay_reset_reason: retention_lost.then_some(TerminalReplayResetReason::RetentionLost), + requested_since_seq: Some(since_seq), + session_ref, + }); + sink(ready); + + // The modes.sync preamble, byte-identical to the legacy block (a + // fresh surface still needs the emulator-mode prelude; the sync + // stays ahead of the pages by admission order — the first page is + // sunk only after this lock is released). + if surface_reset == Some(true) { + let data = s.modes.synthesize(); + if !data.is_empty() { + sink(ServerMessage::TerminalModesSync(TerminalModesSync { + terminal_id: terminal_id.to_string(), + attach_request_id: arid.clone(), + stream_id: s.stream_id.clone(), + data, + })); + } + } + + // The negotiated retention gap: ordered ahead of the pages by + // admission (sunk here under the lock; the pages are sunk after it). + if retention_lost { + sink(ServerMessage::TerminalOutputGap(TerminalOutputGap { + terminal_id: terminal_id.to_string(), + stream_id: s.stream_id.clone(), + attach_request_id: Some(arid.clone()), + from_seq: effective_requested + 1, + to_seq: oldest - 1, + reason: TerminalOutputGapReason::ReplayWindowExceeded, + head_seq: Some(head_seq), + oldest_retained_seq: Some(oldest), + })); + } + + // Select the first page (bounded by the registry's page budget), + // cloning ONLY the selected frames — never the whole ring. + let budget = self.paced_page_max_bytes(); + let first_page = if baseline < head_seq { + paced_page_build( + &s, + conn_id, + baseline, + head_seq, + budget, + OutputSource::Replay, + ) + .map(|build| build.messages) + .unwrap_or_default() + } else { + Vec::new() + }; + let page_end = first_page + .last() + .and_then(page_last_seq) + .unwrap_or(baseline.max(head_seq)); + let page_bytes = first_page + .iter() + .map(|m| serde_json::to_string(m).map(|j| j.len()).unwrap_or(0)) + .sum::() as u64; + + AttachOutcome { + found: true, + geometry, + paced: Some(PacedAttachStart { + session: PacedSessionDesc { + terminal_id: terminal_id.to_string(), + stream_id: s.stream_id.clone(), + attach_request_id: arid, + target: head_seq, + effective_since: baseline, + page_end, + page_bytes, + }, + first_page, + }), } } @@ -1840,6 +2200,156 @@ impl TerminalRegistry { }) } + /// Paced replay page read (responsive-terminal-restore Workstream 1): + /// ONE bounded, ascending page of the `(from_seq, target]` window for a + /// deferred subscriber's session, taking the per-terminal lock briefly + /// (never across pages). The page is packed until the serialized budget + /// (envelope + escaping + batch metadata, via the batch builder's + /// accounting) is reached; a single frame whose own envelope exceeds the + /// budget forms its own atomic single-frame page. Frames are selected + /// BEFORE cloning — no full-ring snapshot per page. + /// + /// `Done` when `from_seq >= target`; `Expired` when retention evicted the + /// next needed frames (exact lost interval `[from_seq+1, new_front-1]`, + /// `resume_from = new_front-1`); `Gone` when the terminal or the + /// subscriber disappeared (cancel the session). + /// + /// Callers must NOT hold the calling connection's writer admission lock + /// (same lock-order rule as [`Self::replay_bounds`]). + pub fn next_replay_page( + &self, + terminal_id: &str, + conn_id: u64, + from_seq: i64, + target: i64, + max_serialized_bytes: i64, + ) -> PacedPage { + let Some(shared) = self.shared_for(terminal_id) else { + return PacedPage::Gone; + }; + let s = shared.lock().expect("terminal lock"); + let Some(sub) = s.subscribers.get(&conn_id) else { + return PacedPage::Gone; + }; + if !sub.paced_terminal_replay_v1 { + return PacedPage::Gone; + } + if from_seq >= target { + return PacedPage::Done; + } + let oldest = s.oldest_retained_seq(); + if from_seq + 1 < oldest { + return PacedPage::Expired { + lost_from: from_seq + 1, + lost_to: oldest - 1, + resume_from: oldest - 1, + head_seq: s.head_seq, + oldest_retained_seq: oldest, + }; + } + match paced_page_build( + &s, + conn_id, + from_seq, + target, + max_serialized_bytes, + OutputSource::Replay, + ) { + Some(build) => PacedPage::Frames { + messages: build.messages, + end_seq: build.end_seq, + serialized_bytes: build.serialized_bytes, + }, + // Defensive: the window is non-empty and retained (checked + // above), so an empty build means nothing the walk could select — + // treat the window as delivered rather than stalling the session. + None => PacedPage::Done, + } + } + + /// Paced replay TAIL page read: the post-replay catch-up phase. Pages the + /// accumulated live range `(from_seq, head]` through the same budget + /// mechanism, and — when the read finds the ring DRAINED (`from_seq >= + /// head`) — clears the subscriber's deferral UNDER THE SAME LOCK HOLD + /// and returns `CaughtUp`: every frame appended before the clear was + /// paged, every frame appended after it fans out directly, so the + /// flag-clear boundary can neither lose nor duplicate a frame, and live + /// output can never overtake an un-sent page (the clear only happens + /// when no page remains un-sunk). + pub fn next_paced_tail_page( + &self, + terminal_id: &str, + conn_id: u64, + from_seq: i64, + max_serialized_bytes: i64, + ) -> PacedTailPage { + let Some(shared) = self.shared_for(terminal_id) else { + return PacedTailPage::Gone; + }; + let mut s = shared.lock().expect("terminal lock"); + let negotiated = match s.subscribers.get(&conn_id) { + Some(sub) => sub.paced_terminal_replay_v1, + None => return PacedTailPage::Gone, + }; + if !negotiated { + return PacedTailPage::Gone; + } + let head_seq = s.head_seq; + if from_seq >= head_seq { + // THE ATOMIC CLEAR: nothing is staged beyond `from_seq`, and the + // last page was sunk before this call — no un-sent page exists, + // so direct fan-out from here on can never overtake a page. + s.subscribers + .get_mut(&conn_id) + .expect("subscriber checked above") + .paced_deferred = false; + return PacedTailPage::CaughtUp; + } + let oldest = s.oldest_retained_seq(); + if from_seq + 1 < oldest { + return PacedTailPage::Expired { + lost_from: from_seq + 1, + lost_to: oldest - 1, + resume_from: oldest - 1, + head_seq, + oldest_retained_seq: oldest, + }; + } + match paced_page_build( + &s, + conn_id, + from_seq, + head_seq, + max_serialized_bytes, + OutputSource::Live, + ) { + Some(build) => PacedTailPage::Frames { + messages: build.messages, + end_seq: build.end_seq, + serialized_bytes: build.serialized_bytes, + }, + None => { + // The ring drained between the head check and the walk (or + // holds nothing past `from_seq`): clear and complete. + s.subscribers + .get_mut(&conn_id) + .expect("subscriber checked above") + .paced_deferred = false; + PacedTailPage::CaughtUp + } + } + } + + /// Resolve a terminal's shared handle under the registry lock, then drop + /// the registry lock (the page reads hold ONLY the per-terminal lock). + fn shared_for(&self, terminal_id: &str) -> Option>> { + let inner = self.inner.lock().expect("registry lock"); + inner + .terminals + .get(terminal_id) + .map(|h| Arc::clone(&h.shared)) + } + /// On socket close: sweep `conn_id` out of EVERY terminal's subscriber set. All /// PTYs keep running (background sessions), reattachable by a future socket. pub fn remove_connection(&self, conn_id: u64) { @@ -3535,6 +4045,13 @@ fn ingest(shared: &Arc>, msg: ServerMessage) { // (source stays 'live'). A single live frame is one small batch — the merge logic // is the same as replay's (proven byte-exact by the deterministic crate goldens). for sub in s.subscribers.values() { + // Restore contract (responsive-terminal-restore): a subscriber with a + // paced session in flight receives NOTHING inline — the retained + // ring is the staging and the session's pages deliver its range in + // seq order (see `Subscriber::paced_deferred`). + if sub.paced_deferred { + continue; + } match ( sub.terminal_output_batch_v1, sub.attach_request_id.as_deref(), @@ -3608,6 +4125,187 @@ fn deliver_batches( } } +// ── Paced replay page production (responsive-terminal-restore, W1) ────────── + +/// One built page: its ascending wire messages, the last seq it covers (the +/// session's new production cursor), and the page's total serialized bytes. +struct PacedPageBuild { + messages: Vec, + end_seq: i64, + serialized_bytes: u64, +} + +/// The last sequence covered by a page wire message. +fn page_last_seq(msg: &ServerMessage) -> Option { + match msg { + ServerMessage::TerminalOutput(o) => Some(o.seq_end), + ServerMessage::TerminalOutputBatch(b) => Some(b.seq_end), + _ => None, + } +} + +/// Conservative per-frame wire-segment estimate for the batch projection: +/// `{"seqStart":N,"seqEnd":M,"endOffset":E,"rawFrameCount":C}` plus the +/// `serializedBytes` field's share. The exact per-segment wire size for +/// realistic seq widths (≤11 digits — a frame per millisecond for a year) +/// stays under this, so a walk that stops at the budget never undercounts. +const SEGMENT_WIRE_OVERHEAD_ESTIMATE: i64 = 96; + +/// Select and project ONE bounded, ascending page of the +/// `(from_seq, to_seq_inclusive]` window for `conn_id`'s subscriber. The +/// caller holds the terminal lock; frames are selected BEFORE cloning (no +/// full-ring snapshot per page). Packing accounts every frame at its +/// STANDALONE envelope cost (the batch builder's own accounting, reusing +/// `measure_serialized_json_bytes` via the incremental scaffold), plus the +/// batch-segment overhead on batch-capable subscribers — an overestimate +/// of the merged wire cost, so every produced page's real serialized bytes +/// stay within the budget. A single frame whose own envelope exceeds the +/// budget forms its own atomic single-frame page (guaranteed progress; the +/// oversize result is explicit, never silently coalesced). +fn paced_page_build( + s: &TerminalShared, + conn_id: u64, + from_seq: i64, + to_seq_inclusive: i64, + budget: i64, + source: OutputSource, +) -> Option { + let sub = s.subscribers.get(&conn_id)?; + let batch_mode = sub.terminal_output_batch_v1 && sub.attach_request_id.is_some(); + let arid = sub.attach_request_id.clone(); + let source_str = match source { + OutputSource::Replay => "replay", + OutputSource::Live => "live", + }; + + // First ring index with seq_start > from_seq (the ring is seq-ascending; + // binary search avoids an O(ring) scan per page). + let (mut lo, mut hi) = (0usize, s.replay.len()); + while lo < hi { + let mid = (lo + hi) / 2; + if s.replay[mid].output.seq_start <= from_seq { + lo = mid + 1; + } else { + hi = mid; + } + } + let start = lo; + if start >= s.replay.len() || s.replay[start].output.seq_start > to_seq_inclusive { + return None; + } + let scaffold = crate::batch::legacy_envelope_scaffold_bytes( + &s.terminal_id, + &s.replay[start].output.stream_id, + arid.as_deref(), + Some(source_str), + ) as i64; + + let mut selected: Vec = Vec::new(); + let mut page_bytes: i64 = 0; + let mut end_seq = from_seq; + // `range` starts at the binary-searched index in O(1) (unlike + // `iter().skip`, which re-advances from the ring's front). + for f in s.replay.range(start..) { + if f.output.seq_start > to_seq_inclusive { + break; + } + let escaped = crate::batch::json_escaped_len(&f.output.data) as i64; + let digits = (crate::batch::digit_count(f.output.seq_start) + + crate::batch::digit_count(f.output.seq_end)) as i64; + let cost = scaffold + + digits + + escaped + + if batch_mode { + SEGMENT_WIRE_OVERHEAD_ESTIMATE + } else { + 0 + }; + if selected.is_empty() { + // Always include the first frame: an over-budget frame forms + // its own atomic single-frame page. + selected.push(f.clone()); + page_bytes = cost; + end_seq = f.output.seq_end; + if cost > budget { + break; + } + continue; + } + if page_bytes + cost > budget { + break; + } + selected.push(f.clone()); + page_bytes += cost; + end_seq = f.output.seq_end; + } + + // Project the selected frames the same way the inline paths deliver them. + let messages: Vec = if batch_mode { + build_batch_messages( + &s.terminal_id, + &selected, + arid.as_deref().unwrap_or(""), + source_str, + ) + } else { + selected + .iter() + .map(|f| { + let mut out = f.output.clone(); + out.attach_request_id = arid.clone(); + out.source = Some(source); + ServerMessage::TerminalOutput(out) + }) + .collect() + }; + let serialized_bytes = messages + .iter() + .map(|m| serde_json::to_string(m).map(|j| j.len()).unwrap_or(0)) + .sum::() as u64; + Some(PacedPageBuild { + messages, + end_seq, + serialized_bytes, + }) +} + +/// The page projection's batch arm: `terminal.output.batch` wire payloads for +/// a bounded page selection (same builder + repacking as +/// [`deliver_batches`], which keeps its per-payload streaming shape for the +/// legacy inline path; pages are budget-bounded, so materializing their +/// messages is bounded by the page budget). +fn build_batch_messages( + terminal_id: &str, + frames: &[RetainedFrame], + attach_request_id: &str, + source: &str, +) -> Vec { + if frames.is_empty() { + return Vec::new(); + } + let batch_max = terminal_stream_batch_max_bytes() as i64; + let inputs: Vec = frames.iter().map(|f| f.to_batch_input()).collect(); + let batches = build_terminal_output_batches(&BatchBuildInput { + frames: &inputs, + max_serialized_bytes: batch_max, + max_total_serialized_bytes: None, + terminal_id: terminal_id.to_string(), + attach_request_id: Some(attach_request_id.to_string()), + source: Some(source.to_string()), + }); + let mut messages = Vec::new(); + for batch in &batches { + for payload in + build_batch_wire_payloads(terminal_id, batch, attach_request_id, source, batch_max) + { + if let Ok(msg) = serde_json::from_value::(payload) { + messages.push(msg); + } + } + } + messages +} + /// b8ke ext r30 F2 (test-only): the rekey's deterministic INTERLOCK — /// parks the rekey's critical section between the old→new coordinator /// move and the retained-claim move for ONE targeted terminal id, so a @@ -4262,6 +4960,7 @@ mod tests { false, None, None, + None, ); let legacy = outputs(&legacy_seen); assert!( @@ -4295,6 +4994,7 @@ mod tests { false, None, None, + None, ); let bs = batches(&batch_seen); assert!( @@ -4342,7 +5042,18 @@ mod tests { reg.feed("T", frame(1, "a\u{1F600}b\r\n", "S")); // a😀b␍␊ let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("m".into()), 0, true, false, None, None); + let _ = reg.attach( + "T", + 1, + sink, + Some("m".into()), + 0, + true, + false, + None, + None, + None, + ); let bs = batches(&seen); assert_eq!(bs.len(), 1); let b = &bs[0]; @@ -4373,6 +5084,7 @@ mod tests { false, None, None, + None, ); assert!(out.found); @@ -4420,6 +5132,7 @@ mod tests { true, None, None, + None, ); let ready = attach_ready(&seen).expect("attach.ready sent"); assert_eq!(ready.head_seq, 2); @@ -4453,6 +5166,7 @@ mod tests { true, None, None, + None, ); let ready = attach_ready(&seen).expect("attach.ready sent"); assert_eq!( @@ -4484,6 +5198,7 @@ mod tests { true, None, None, + None, ); let ready = attach_ready(&seen).expect("attach.ready sent"); assert_eq!(ready.head_seq, 0); @@ -4531,43 +5246,949 @@ mod tests { assert_eq!(reg.replay_bounds("nope"), None); } + // ── Paced replay core (responsive-terminal-restore, Workstream 1) ──────── + // + // The registry's page-read primitives + the attach-time session start. + // The ws pacing coordinator (credit gating, tail drain, events) drives + // these; its behavior is pinned by the `freshell-ws` integration suite. + + /// Flatten one page's wire messages into `(seq, data)` pairs (legacy + /// per-frame and batch pages both reassemble to these). + fn page_seq_data(messages: &[ServerMessage]) -> Vec<(i64, String)> { + let mut out = Vec::new(); + for msg in messages { + match msg { + ServerMessage::TerminalOutput(o) => out.push((o.seq_start, o.data.clone())), + ServerMessage::TerminalOutputBatch(b) => { + // A merged batch is one seq span over its concatenated data. + let mut prev = 0i64; + for seg in &b.segments { + let chunk = crate::batch::slice_utf16(&b.data, prev, seg.end_offset); + out.push((seg.seq_start, chunk)); + prev = seg.end_offset; + } + } + other => panic!("unexpected page message: {other:?}"), + } + } + out + } + + fn page_serialized_bytes(messages: &[ServerMessage]) -> usize { + messages + .iter() + .map(|m| serde_json::to_string(m).expect("page serializes").len()) + .sum() + } + + /// The subscriber-side view of one paced attach: sink messages seen so + /// far (should be ONLY the control prelude — ready/sync/gap) plus the + /// returned first page and session description. #[test] - fn unpaced_attach_ready_pins_the_exact_wire_keys_for_both_negotiation_sides() { - // Compatibility invariant (load-bearing): a connection that did NOT - // negotiate sees a ready frame byte-identical to the pre-contract - // shape — `oldestRetainedSeq` is ABSENT from the wire (not null), and - // `replayResetReason` is still absent. The negotiated side adds - // exactly one key: `oldestRetainedSeq`. + fn paced_attach_returns_the_first_page_instead_of_an_inline_replay() { let reg = TerminalRegistry::new(); + reg.set_paced_page_max_bytes(1024); reg.insert_headless("T", "S"); - reg.feed("T", frame(1, "one\r\n", "S")); + for seq in 1..=8 { + reg.feed("T", frame(seq, &format!("data-{seq:03}\r\n"), "S")); + } - let (plain_sink, plain_seen) = collector(); - let _ = reg.attach( + let (sink, seen) = collector(); + let out = reg.attach( "T", 1, - plain_sink, - Some("plain".into()), + sink, + Some("paced-1".into()), 0, false, - false, + true, + None, None, None, ); - let plain_ready = attach_ready(&plain_seen).expect("attach.ready sent"); - assert_eq!(plain_ready.oldest_retained_seq, None); - let json = serde_json::to_value(ServerMessage::TerminalAttachReady(plain_ready)).unwrap(); - let mut keys: Vec<&str> = json - .as_object() - .unwrap() - .keys() - .map(String::as_str) - .collect(); - keys.sort_unstable(); - assert_eq!( - keys, - vec![ - "attachRequestId", + assert!(out.found); + let start = out.paced.expect("negotiated attach starts a paced session"); + assert_eq!(start.session.terminal_id, "T"); + assert_eq!(start.session.stream_id, "S"); + assert_eq!(start.session.attach_request_id, "paced-1"); + assert_eq!(start.session.target, 8, "target is the head at attach time"); + assert_eq!(start.session.effective_since, 0); + + // The inline sink saw ONLY the control prelude — the replay frames + // travel as returned pages, never sunk under the attach lock. + for msg in seen.lock().unwrap().iter() { + assert!( + !matches!(msg, ServerMessage::TerminalOutput(_)), + "no replay output may be sunk inline: {msg:?}" + ); + } + + // The first page is a bounded ascending prefix of the replay window. + assert!(!start.first_page.is_empty(), "there is replay to page"); + assert!(page_serialized_bytes(&start.first_page) <= 1024); + let page1 = page_seq_data(&start.first_page); + assert_eq!( + page1.first().unwrap().0, + 1, + "the page starts at the baseline+1" + ); + let last_seq = page1.last().unwrap().0; + assert_eq!( + start.session.page_end, last_seq, + "the session cursor is the first page's last seq" + ); + assert!( + last_seq < 8, + "the first page is a bounded prefix, not the whole window" + ); + assert_eq!( + start.session.page_bytes, + page_serialized_bytes(&start.first_page) as u64 + ); + for msg in &start.first_page { + match msg { + ServerMessage::TerminalOutput(o) => { + assert_eq!(o.attach_request_id.as_deref(), Some("paced-1")); + assert_eq!( + o.source, + Some(OutputSource::Replay), + "replay pages are stamped source:'replay'" + ); + } + other => panic!("unexpected first-page message: {other:?}"), + } + } + } + + /// A batch-capable negotiated subscriber gets `terminal.output.batch` + /// pages that reassemble to the same bytes as the per-frame projection. + #[test] + fn paced_batch_pages_reassemble_to_the_frame_bytes() { + let reg = TerminalRegistry::new(); + reg.set_paced_page_max_bytes(4096); + reg.insert_headless("T", "S"); + for seq in 1..=5 { + reg.feed("T", frame(seq, &format!("line-{seq}\r\n"), "S")); + } + let (sink, _seen) = collector(); + let out = reg.attach( + "T", + 2, + sink, + Some("batch-paced".into()), + 0, + true, + true, + None, + None, + None, + ); + let start = out.paced.expect("paced session"); + assert!( + start + .first_page + .iter() + .all(|m| matches!(m, ServerMessage::TerminalOutputBatch(_))), + "a batch-capable subscriber gets batch pages" + ); + // Drive any remaining pages (the first page may be a bounded prefix) + // and reassemble EVERYTHING the session delivers. + let mut page_messages = start.first_page.clone(); + let mut cursor = start.session.page_end; + while cursor < start.session.target { + match reg.next_replay_page("T", 2, cursor, start.session.target, 4096) { + PacedPage::Frames { + messages, end_seq, .. + } => { + page_messages.extend(messages); + cursor = end_seq; + } + other => panic!("unexpected replay read: {other:?}"), + } + } + let reassembled: String = { + let mut v: Vec<(i64, String)> = page_seq_data(&page_messages); + v.sort_by_key(|(s, _)| *s); + v.into_iter().map(|(_, d)| d).collect() + }; + assert_eq!( + reassembled, + (1..=5).map(|i| format!("line-{i}\r\n")).collect::() + ); + } + + /// Pages ascend within the serialized budget to the FIXED attach-time + /// target — output produced after the attach never extends the replay + /// range (it is tail-phase delivery instead). + #[test] + fn paced_replay_pages_stop_at_the_fixed_target_within_the_budget() { + let reg = TerminalRegistry::new(); + reg.set_paced_page_max_bytes(1024); + reg.insert_headless("T", "S"); + for seq in 1..=8 { + reg.feed("T", frame(seq, &format!("data-{seq:03}\r\n"), "S")); + } + let (sink, _seen) = collector(); + let out = reg.attach( + "T", + 1, + sink, + Some("paced".into()), + 0, + false, + true, + None, + None, + None, + ); + let start = out.paced.expect("paced session"); + let mut cursor = start.session.page_end; + let mut collected = page_seq_data(&start.first_page); + + // Concurrent output AFTER the attach must NOT extend the target. + for seq in 9..=12 { + reg.feed("T", frame(seq, &format!("post-{seq:03}\r\n"), "S")); + } + + while cursor < 8 { + match reg.next_replay_page("T", 1, cursor, 8, 1024) { + PacedPage::Frames { + messages, + end_seq, + serialized_bytes, + } => { + assert!( + serialized_bytes as usize <= 1024, + "every replay page honors the serialized budget" + ); + assert!(end_seq > cursor, "pages must make progress"); + assert!(end_seq <= 8, "pages never pass the fixed target"); + collected.extend(page_seq_data(&messages)); + cursor = end_seq; + } + PacedPage::Done => panic!("Done while cursor {cursor} < target 8"), + PacedPage::Expired { .. } => panic!("no retention loss in this fixture"), + PacedPage::Gone => panic!("terminal vanished mid-replay"), + } + } + assert_eq!(cursor, 8); + match reg.next_replay_page("T", 1, cursor, 8, 1024) { + PacedPage::Done => {} + other => panic!("a drained replay window reads Done, got {other:?}"), + } + let replayed: String = { + let mut v = collected.clone(); + v.sort_by_key(|(s, _)| *s); + v.into_iter().map(|(_, d)| d).collect() + }; + assert_eq!( + replayed, + (1..=8) + .map(|i| format!("data-{i:03}\r\n")) + .collect::(), + "the replay pages cover the window exactly, in order, no loss/dup" + ); + + // The post-attach frames are TAIL delivery: pages from the target to + // the current head, then the deferral clears (CaughtUp). + let mut tail_cursor = cursor; + let mut tail_frames = Vec::new(); + loop { + match reg.next_paced_tail_page("T", 1, tail_cursor, 1024) { + PacedTailPage::Frames { + messages, + end_seq, + serialized_bytes, + } => { + assert!(serialized_bytes as usize <= 1024); + assert!(end_seq > tail_cursor); + tail_frames.extend(page_seq_data(&messages)); + tail_cursor = end_seq; + } + PacedTailPage::Expired { .. } => panic!("no retention loss in this fixture"), + PacedTailPage::CaughtUp => break, + PacedTailPage::Gone => panic!("terminal vanished mid-tail"), + } + } + assert_eq!(tail_cursor, 12, "the tail drains to the current head"); + let tail: String = { + let mut v = tail_frames; + v.sort_by_key(|(s, _)| *s); + v.into_iter().map(|(_, d)| d).collect() + }; + assert_eq!( + tail, + (9..=12) + .map(|i| format!("post-{i:03}\r\n")) + .collect::(), + "the tail delivers exactly the post-attach range" + ); + } + + /// A single frame whose own envelope exceeds the page budget forms its + /// own atomic single-frame page (guaranteed progress, never split, never + /// silently coalesced into a "budget" page). + #[test] + fn paced_replay_oversized_frame_forms_its_own_atomic_page() { + let reg = TerminalRegistry::new(); + reg.set_paced_page_max_bytes(256); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "tiny-a\r\n", "S")); + reg.feed("T", frame(2, &"X".repeat(2048), "S")); + reg.feed("T", frame(3, "tiny-b\r\n", "S")); + + let (sink, _seen) = collector(); + let out = reg.attach( + "T", + 1, + sink, + Some("paced".into()), + 0, + false, + true, + None, + None, + None, + ); + let start = out.paced.expect("paced session"); + assert_eq!( + start.session.page_end, 1, + "the first page stops before the oversize frame" + ); + + match reg.next_replay_page("T", 1, 1, 3, 256) { + PacedPage::Frames { + messages, + end_seq, + serialized_bytes, + } => { + assert_eq!(end_seq, 2); + assert_eq!( + messages.len(), + 1, + "the oversize frame is ONE atomic message" + ); + assert!( + serialized_bytes as usize > 256, + "the oversize frame honestly exceeds the budget as its own page" + ); + match &messages[0] { + ServerMessage::TerminalOutput(o) => { + assert_eq!(o.seq_start, 2); + assert_eq!(o.data.len(), 2048); + } + other => panic!("per-frame subscriber gets terminal.output: {other:?}"), + } + } + other => panic!("expected the atomic oversize page, got {other:?}"), + } + match reg.next_replay_page("T", 1, 2, 3, 256) { + PacedPage::Frames { + messages, end_seq, .. + } => { + assert_eq!(end_seq, 3); + assert_eq!( + page_seq_data(&messages), + vec![(3, "tiny-b\r\n".to_string())] + ); + } + other => panic!("expected the final page, got {other:?}"), + } + } + + /// Retention expiry mid-replay: the exact lost interval and the resume + /// position, both consistent with the ring's live bounds. + #[test] + fn paced_replay_expired_reports_the_exact_interval_and_resume() { + let reg = TerminalRegistry::new(); + // Tiny CHAR ring so feeding evicts the front deterministically. + reg.set_scrollback_max_bytes(60); + reg.insert_headless("T", "S"); + reg.set_paced_page_max_bytes(0); // per-frame pages: deterministic cursor control + for seq in 1..=3 { + reg.feed("T", frame(seq, "chunk123\r\n", "S")); // 10 chars each + } + let (sink, _seen) = collector(); + let out = reg.attach( + "T", + 1, + sink, + Some("paced".into()), + 0, + false, + true, + None, + None, + None, + ); + let start = out.paced.expect("paced session"); + assert_eq!(start.session.page_end, 1, "budget 0 => one frame per page"); + + // Evict frames 2..: 10 more chunks (100 chars) pushes the front well + // past the session cursor. + for seq in 4..=13 { + reg.feed("T", frame(seq, "chunk123\r\n", "S")); + } + let bounds = reg.replay_bounds("T").expect("bounds"); + assert!( + bounds.oldest_retained_seq > 2, + "the fixture evicted the frames the session needs next" + ); + + match reg.next_replay_page("T", 1, 1, bounds.head_seq, 0) { + PacedPage::Expired { + lost_from, + lost_to, + resume_from, + head_seq, + oldest_retained_seq, + } => { + assert_eq!(lost_from, 2, "the lost interval starts at cursor+1"); + assert_eq!( + lost_to, + bounds.oldest_retained_seq - 1, + "the lost interval ends just before the new ring front" + ); + assert_eq!(resume_from, bounds.oldest_retained_seq - 1); + assert_eq!(head_seq, bounds.head_seq); + assert_eq!(oldest_retained_seq, bounds.oldest_retained_seq); + } + other => panic!("expected Expired, got {other:?}"), + } + + // Continuation from the new baseline: the tail pages everything the + // ring still holds, then the deferral clears. + let mut tail_cursor = bounds.oldest_retained_seq - 1; + let mut drained = Vec::new(); + loop { + match reg.next_paced_tail_page("T", 1, tail_cursor, 0) { + PacedTailPage::Frames { + messages, end_seq, .. + } => { + drained.extend(page_seq_data(&messages)); + tail_cursor = end_seq; + } + PacedTailPage::CaughtUp => break, + other => panic!("unexpected tail read: {other:?}"), + } + } + assert_eq!(tail_cursor, bounds.head_seq); + let drained_seqs: Vec = drained.iter().map(|(s, _)| *s).collect(); + assert_eq!( + drained_seqs, + (bounds.oldest_retained_seq..=bounds.head_seq).collect::>(), + "the continuation covers exactly the retained range" + ); + } + + /// While a paced session is active the subscriber's live output is NOT + /// sunk by ingest (the ring is the staging); after the session catches + /// up (deferral cleared under the tail read's lock), ingest resumes + /// direct delivery. Concurrent production across the flag-clear + /// boundary is delivered exactly once — paged or direct, never both, + /// never neither. + #[test] + fn paced_deferral_stages_live_output_and_resumes_direct_delivery_exactly_once() { + let reg = TerminalRegistry::new(); + reg.set_paced_page_max_bytes(0); // per-frame pages + reg.insert_headless("T", "S"); + for seq in 1..=3 { + reg.feed("T", frame(seq, "early-1\r\n", "S")); + } + let (sink, seen) = collector(); + let out = reg.attach( + "T", + 1, + sink, + Some("paced".into()), + 0, + false, + true, + None, + None, + None, + ); + let start = out.paced.expect("paced session"); + assert_eq!(start.session.page_end, 1); + + // Live output while the session is active: staged in the ring, NOT + // sunk. + reg.feed("T", frame(4, "live-04\r\n", "S")); + assert!( + seen.lock() + .unwrap() + .iter() + .all(|m| !matches!(m, ServerMessage::TerminalOutput(_))), + "deferred subscriber receives nothing inline" + ); + + // Concurrent production racing the tail drain. + let feeder_reg = reg.clone(); + let feeder = std::thread::spawn(move || { + for seq in 5..=40 { + feeder_reg.feed("T", frame(seq, &format!("race-{seq:02}\r\n"), "S")); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + }); + + // Drain: replay pages to the target, then tail pages to CaughtUp. + let mut collected = page_seq_data(&start.first_page); + let mut cursor = start.session.page_end; + while cursor < start.session.target { + match reg.next_replay_page("T", 1, cursor, start.session.target, 0) { + PacedPage::Frames { + messages, end_seq, .. + } => { + collected.extend(page_seq_data(&messages)); + cursor = end_seq; + } + other => panic!("unexpected replay read: {other:?}"), + } + } + loop { + match reg.next_paced_tail_page("T", 1, cursor, 0) { + PacedTailPage::Frames { + messages, end_seq, .. + } => { + collected.extend(page_seq_data(&messages)); + cursor = end_seq; + } + PacedTailPage::CaughtUp => break, + other => panic!("unexpected tail read: {other:?}"), + } + } + feeder.join().expect("feeder joins"); + + // Post-clear production flows DIRECTLY through ingest again. The + // feeder's tail may have landed either side of the clear (any split + // is valid); frame 41 is fed strictly AFTER the drain, so it must be + // a DIRECT delivery. + reg.feed("T", frame(41, "after-41\r\n", "S")); + let direct: Vec<(i64, String)> = seen + .lock() + .unwrap() + .iter() + .filter_map(|m| match m { + ServerMessage::TerminalOutput(o) => Some((o.seq_start, o.data.clone())), + _ => None, + }) + .collect(); + assert_eq!( + direct.last(), + Some(&(41, "after-41\r\n".to_string())), + "post-clear output is delivered directly by ingest: {direct:?}" + ); + + // The no-loss/no-dup invariant across the flag-clear boundary: + // pages + direct deliveries together are exactly frames 1..=41, once. + let mut all: Vec<(i64, String)> = collected; + all.extend(direct); + all.sort_by_key(|(s, _)| *s); + let seqs: Vec = all.iter().map(|(s, _)| *s).collect(); + assert_eq!( + seqs, + (1..=41).collect::>(), + "every produced frame delivered exactly once, in seq order" + ); + assert_eq!(all.iter().map(|(_, d)| d.clone()).collect::(), { + let mut s = String::new(); + for _seq in 1..=3 { + s.push_str("early-1\r\n"); + } + s.push_str("live-04\r\n"); + for seq in 5..=40 { + s.push_str(&format!("race-{seq:02}\r\n")); + } + s.push_str("after-41\r\n"); + s + }); + } + + /// A re-attach replaces the subscriber — an armed paced deferral does not + /// survive into the new subscription unless the new attach arms its own. + #[test] + fn reattach_replaces_the_paced_deferral() { + let reg = TerminalRegistry::new(); + reg.set_paced_page_max_bytes(0); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "one\r\n", "S")); + + let (sink, seen) = collector(); + let out = reg.attach( + "T", + 1, + sink.clone(), + Some("paced-a".into()), + 0, + false, + true, + None, + None, + None, + ); + assert!(out.paced.is_some(), "the paced attach arms the deferral"); + reg.feed("T", frame(2, "two\r\n", "S")); + assert!( + seen.lock() + .unwrap() + .iter() + .all(|m| !matches!(m, ServerMessage::TerminalOutput(_))), + "staged while the first session is active" + ); + + // A non-paced re-attach (the fallback shape) cancels the session's + // deferral: the legacy re-attach replays the window INLINE (frames 1 + // and 2 — including the one staged while deferred), and the NEXT + // produced frame flows directly by ingest, with no pages to drive. + let out2 = reg.attach( + "T", + 1, + sink.clone(), + Some("legacy-b".into()), + 0, + false, + false, + None, + None, + None, + ); + assert!( + out2.paced.is_none(), + "a non-negotiated re-attach stays legacy" + ); + reg.feed("T", frame(3, "three\r\n", "S")); + let live: Vec = seen + .lock() + .unwrap() + .iter() + .filter_map(|m| match m { + ServerMessage::TerminalOutput(o) if o.source == Some(OutputSource::Live) => { + Some(o.data.clone()) + } + _ => None, + }) + .collect(); + assert_eq!( + live, + vec!["three\r\n".to_string()], + "after the deferral clears, live output flows directly" + ); + + // A paced re-attach arms a FRESH session whose first page includes + // everything staged since its own baseline. + let out3 = reg.attach( + "T", + 1, + sink.clone(), + Some("paced-c".into()), + 0, + false, + true, + None, + None, + None, + ); + let start = out3 + .paced + .expect("the paced re-attach starts a fresh session"); + assert_eq!(start.session.attach_request_id, "paced-c"); + assert_eq!(start.session.target, 3); + // Budget 0 => per-frame pages: drive the fresh session to completion + // and assert it pages the WHOLE window (including frame 2, staged + // while the first session was deferred). + let mut paged: Vec<(i64, String)> = page_seq_data(&start.first_page); + let mut cursor = start.session.page_end; + while cursor < start.session.target { + match reg.next_replay_page("T", 1, cursor, start.session.target, 0) { + PacedPage::Frames { + messages, end_seq, .. + } => { + paged.extend(page_seq_data(&messages)); + cursor = end_seq; + } + other => panic!("unexpected replay read: {other:?}"), + } + } + let mut paged_seqs: Vec = paged.iter().map(|(s, _)| *s).collect(); + paged_seqs.sort_unstable(); + assert_eq!( + paged_seqs, + vec![1, 2, 3], + "the fresh session pages the whole window" + ); + } + + /// Retention loss AT ATTACH (requested since predates the retained + /// ring): the negotiated connection gets the retention gap with the + /// task-2 bounds fields, `replayResetReason: retention_lost`, and an + /// effective baseline of `oldest-1` — the session continues from what + /// is retained (nothing is killed, nothing stalls). + #[test] + fn paced_attach_with_retention_loss_emits_the_negotiated_gap_and_resets_the_baseline() { + let reg = TerminalRegistry::new(); + reg.set_scrollback_max_bytes(60); + reg.insert_headless("T", "S"); + for seq in 1..=13 { + reg.feed("T", frame(seq, "chunk123\r\n", "S")); + } + let bounds = reg.replay_bounds("T").expect("bounds"); + assert!( + bounds.oldest_retained_seq > 1, + "the front has evicted past seq 1" + ); + + let (sink, seen) = collector(); + let out = reg.attach( + "T", + 1, + sink, + Some("paced".into()), + 0, + false, + true, + None, + None, + None, + ); + assert!(out.found); + let start = out + .paced + .expect("paced session continues from what is retained"); + assert_eq!( + start.session.effective_since, + bounds.oldest_retained_seq - 1, + "the baseline resets to oldest-1" + ); + assert_eq!(start.session.target, bounds.head_seq); + let first = page_seq_data(&start.first_page); + assert_eq!( + first.first().unwrap().0, + bounds.oldest_retained_seq, + "the first page starts at the retained front" + ); + + let ready = attach_ready(&seen).expect("attach.ready sent"); + assert_eq!( + ready.replay_reset_reason, + Some(TerminalReplayResetReason::RetentionLost), + "the ready frame names the retention reset" + ); + assert_eq!(ready.oldest_retained_seq, Some(bounds.oldest_retained_seq)); + assert_eq!( + ready.effective_since_seq, + Some(bounds.oldest_retained_seq - 1) + ); + assert_eq!(ready.requested_since_seq, Some(0)); + assert_eq!(ready.replay_from_seq, bounds.oldest_retained_seq); + assert_eq!(ready.replay_to_seq, bounds.head_seq); + + let gap = seen + .lock() + .unwrap() + .iter() + .find_map(|m| match m { + ServerMessage::TerminalOutputGap(g) => Some(g.clone()), + _ => None, + }) + .expect("the retention gap is sunk with the ready prelude"); + assert_eq!( + gap.reason, + freshell_protocol::TerminalOutputGapReason::ReplayWindowExceeded + ); + assert_eq!( + gap.from_seq, 1, + "the lost interval starts at the requested baseline+1" + ); + assert_eq!(gap.to_seq, bounds.oldest_retained_seq - 1); + assert_eq!(gap.attach_request_id.as_deref(), Some("paced")); + assert_eq!(gap.head_seq, Some(bounds.head_seq)); + assert_eq!(gap.oldest_retained_seq, Some(bounds.oldest_retained_seq)); + } + + /// The compatibility twin: a NON-negotiated attach with the same + /// retention loss keeps today's silent behavior — no gap frame, no reset + /// reason, no retention bounds, replay silently starting at the ring + /// front. + #[test] + fn plain_attach_with_retention_loss_stays_silent_and_byte_identical() { + let reg = TerminalRegistry::new(); + reg.set_scrollback_max_bytes(60); + reg.insert_headless("T", "S"); + for seq in 1..=13 { + reg.feed("T", frame(seq, "chunk123\r\n", "S")); + } + let bounds = reg.replay_bounds("T").expect("bounds"); + + let (sink, seen) = collector(); + let out = reg.attach( + "T", + 1, + sink, + Some("plain".into()), + 0, + false, + false, + None, + None, + None, + ); + assert!( + out.paced.is_none(), + "non-negotiated never starts a paced session" + ); + let ready = attach_ready(&seen).expect("attach.ready sent"); + assert_eq!(ready.replay_reset_reason, None); + assert_eq!(ready.oldest_retained_seq, None); + assert_eq!(ready.replay_from_seq, bounds.oldest_retained_seq); + let frames = outputs(&seen); + assert_eq!( + frames.first().map(|f| f.seq_start), + Some(bounds.oldest_retained_seq), + "legacy replay silently starts at the retained front" + ); + assert!( + !seen + .lock() + .unwrap() + .iter() + .any(|m| matches!(m, ServerMessage::TerminalOutputGap(_))), + "no gap frame for a non-negotiated connection" + ); + } + + /// A negotiated attach to an ALREADY-EXITED terminal keeps the legacy + /// inline path (frozen-tail replay + synthetic exit, in their legacy + /// order): pacing arms only for Running terminals this increment. + #[test] + fn paced_negotiation_on_an_exited_terminal_keeps_the_legacy_inline_path() { + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "tail\r\n", "S")); + assert!(reg.finish_pty_exit("T", 0)); + + let (sink, seen) = collector(); + let out = reg.attach( + "T", + 1, + sink, + Some("paced".into()), + 0, + false, + true, + None, + None, + None, + ); + assert!( + out.paced.is_none(), + "exited terminals stay on the legacy path" + ); + let frames = outputs(&seen); + assert_eq!(frames.len(), 1, "the frozen tail replays inline"); + assert_eq!(frames[0].source, Some(OutputSource::Replay)); + assert!( + seen.lock() + .unwrap() + .iter() + .any(|m| matches!(m, ServerMessage::TerminalExit(_))), + "the synthetic exit follows the inline replay" + ); + } + + /// TERM-07 seam: the attach-threaded `maxReplayBytes` is recorded on the + /// subscriber with NO delivery-behavior change (it stays unread for + /// delivery decisions this increment; the increment-3 snapshot work + /// consumes it). + #[test] + fn attach_records_max_replay_bytes_on_the_subscriber() { + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "one\r\n", "S")); + let (sink, _seen) = collector(); + let _ = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + Some(128 * 1024), + ); + let recorded = { + let inner = reg.inner.lock().unwrap(); + let handle = inner.terminals.get("T").unwrap(); + let s = handle.shared.lock().unwrap(); + s.subscribers + .get(&1) + .expect("subscriber installed") + .max_replay_bytes + }; + assert_eq!(recorded, Some(128 * 1024)); + + // Absent stays absent; the legacy path threads it identically. + let (sink2, _seen2) = collector(); + let _ = reg.attach( + "T", + 2, + sink2, + Some("b".into()), + 0, + false, + false, + None, + None, + None, + ); + let recorded2 = { + let inner = reg.inner.lock().unwrap(); + let handle = inner.terminals.get("T").unwrap(); + let s = handle.shared.lock().unwrap(); + s.subscribers.get(&2).expect("subscriber").max_replay_bytes + }; + assert_eq!(recorded2, None); + } + + #[test] + fn unpaced_attach_ready_pins_the_exact_wire_keys_for_both_negotiation_sides() { + // Compatibility invariant (load-bearing): a connection that did NOT + // negotiate sees a ready frame byte-identical to the pre-contract + // shape — `oldestRetainedSeq` is ABSENT from the wire (not null), and + // `replayResetReason` is still absent. The negotiated side adds + // exactly one key: `oldestRetainedSeq`. + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + reg.feed("T", frame(1, "one\r\n", "S")); + + let (plain_sink, plain_seen) = collector(); + let _ = reg.attach( + "T", + 1, + plain_sink, + Some("plain".into()), + 0, + false, + false, + None, + None, + None, + ); + let plain_ready = attach_ready(&plain_seen).expect("attach.ready sent"); + assert_eq!(plain_ready.oldest_retained_seq, None); + let json = serde_json::to_value(ServerMessage::TerminalAttachReady(plain_ready)).unwrap(); + let mut keys: Vec<&str> = json + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + keys.sort_unstable(); + assert_eq!( + keys, + vec![ + "attachRequestId", "effectiveSinceSeq", "geometryAuthority", "geometryEpoch", @@ -4593,6 +6214,7 @@ mod tests { true, None, None, + None, ); let paced_ready = attach_ready(&paced_seen).expect("attach.ready sent"); assert_eq!(paced_ready.oldest_retained_seq, Some(1)); @@ -4640,6 +6262,7 @@ mod tests { false, None, None, + None, ); reg.feed("T", frame(1, "before\r\n", "S")); assert_eq!(outputs(&seen_a).len(), 1); @@ -4666,6 +6289,7 @@ mod tests { false, None, None, + None, ); let replayed = outputs(&seen_b); assert_eq!( @@ -4691,6 +6315,7 @@ mod tests { false, None, None, + None, ); // Second attach: geometry authority flips to multi_client_unknown. let _ = reg.attach( @@ -4703,6 +6328,7 @@ mod tests { false, None, None, + None, ); let ready_b = attach_ready(&seen_b).unwrap(); assert_eq!( @@ -4737,6 +6363,7 @@ mod tests { false, None, None, + None, ); for i in 1..=5 { reg.feed("T", frame(i, &format!("line-{i}\r\n"), "S")); @@ -4757,6 +6384,7 @@ mod tests { false, None, None, + None, ); let ready = attach_ready(&seen_r).unwrap(); assert_eq!(ready.effective_since_seq, Some(3)); @@ -4776,7 +6404,18 @@ mod tests { reg.feed("T", frame(1, "old\r\n", "S")); let (sink, seen) = collector(); - let _ = reg.attach("T", 7, sink, Some("z".into()), 0, false, false, None, None); + let _ = reg.attach( + "T", + 7, + sink, + Some("z".into()), + 0, + false, + false, + None, + None, + None, + ); // A live frame produced AFTER attach must arrive after the replayed one. reg.feed("T", frame(2, "new\r\n", "S")); @@ -4798,7 +6437,7 @@ mod tests { fn attach_to_unknown_terminal_reports_not_found() { let reg = TerminalRegistry::new(); let (sink, seen) = collector(); - let out = reg.attach("nope", 1, sink, None, 0, false, false, None, None); + let out = reg.attach("nope", 1, sink, None, 0, false, false, None, None, None); assert!(!out.found); assert!(seen.lock().unwrap().is_empty()); } @@ -4828,7 +6467,18 @@ mod tests { reg.insert_headless("T", "S"); let rev_before = reg.revision(); let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); + let _ = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); assert!(reg.kill("T")); assert!(!reg.is_running("T"), "killed terminal is removed"); @@ -4856,8 +6506,8 @@ mod tests { reg.insert_headless("T-b", "S2"); let (sink_a, seen_a) = collector(); let (sink_b, seen_b) = collector(); - let _ = reg.attach("T-a", 1, sink_a, None, 0, false, false, None, None); - let _ = reg.attach("T-b", 2, sink_b, None, 0, false, false, None, None); + let _ = reg.attach("T-a", 1, sink_a, None, 0, false, false, None, None, None); + let _ = reg.attach("T-b", 2, sink_b, None, 0, false, false, None, None, None); let rev_before = reg.revision(); let killed = reg.kill_all(); @@ -5143,7 +6793,18 @@ mod tests { assert!(!dir[0].has_clients); let (sink, _seen) = collector(); - let _ = reg.attach("T", 9, sink, Some("a".into()), 0, false, false, None, None); + let _ = reg.attach( + "T", + 9, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); assert!(reg.directory()[0].has_clients); reg.detach("T", 9); assert!(!reg.directory()[0].has_clients); @@ -5264,6 +6925,7 @@ mod tests { false, None, None, + None, TerminalAttachIntent::ViewportHydrate, 131, 48, @@ -5286,6 +6948,7 @@ mod tests { false, None, None, + None, TerminalAttachIntent::TransportReconnect, 67, 30, @@ -5338,6 +7001,7 @@ mod tests { false, None, None, + None, ); // A secondary viewer must be able to attach without silently taking @@ -5356,6 +7020,7 @@ mod tests { false, None, None, + None, ); // Later attach generations from that same second socket are still @@ -5419,8 +7084,19 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink, _seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); // conn 1 is attached - // conn 2 reconnects with another socket attached and no prior attachment of its own. + let _ = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); // conn 1 is attached + // conn 2 reconnects with another socket attached and no prior attachment of its own. let out = reg.resize_for_attach("T", 2, TerminalAttachIntent::TransportReconnect, 95, 41); assert_eq!(out, AttachResizeStatus::Skipped); assert_eq!(reg.geometry("T"), Some((120, 30, 1))); @@ -5441,6 +7117,7 @@ mod tests { false, None, None, + None, ); // The first transport reconnect from B is replay-only while A views @@ -5459,6 +7136,7 @@ mod tests { false, None, None, + None, ); let out = reg.resize_for_attach("T", 2, TerminalAttachIntent::TransportReconnect, 95, 41); @@ -5508,6 +7186,7 @@ mod tests { false, None, None, + None, ); let _ = reg.attach( "T2", @@ -5519,6 +7198,7 @@ mod tests { false, None, None, + None, ); reg.remove_connection(42); @@ -5545,7 +7225,18 @@ mod tests { assert!(reg.finish_pty_exit("T", 7)); let (sink, seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); + let outcome = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); assert!(outcome.found); let exit = seen.lock().unwrap().iter().find_map(|m| match m { @@ -5599,6 +7290,7 @@ mod tests { false, None, Some(true), + None, ); assert!(outcome.found); @@ -5649,6 +7341,7 @@ mod tests { false, None, Some(true), + None, ); assert!(outcome.found); @@ -5689,6 +7382,7 @@ mod tests { false, None, None, + None, ); // … and explicitly false: no sync either way (fixture f14's gating). let (sink_b, seen_b) = collector(); @@ -5702,6 +7396,7 @@ mod tests { false, None, Some(false), + None, ); assert!(modes_syncs(&seen_a).is_empty(), "flag absent => no sync"); @@ -5727,6 +7422,7 @@ mod tests { false, None, Some(true), + None, ); assert!(modes_syncs(&seen).is_empty(), "empty synthesis => no sync"); } @@ -5740,7 +7436,7 @@ mod tests { // The client fails closed on a sync lacking attachRequestId // (`missing_attach_request_id`), so the server never builds one. let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, None, 0, false, false, None, Some(true)); + let _ = reg.attach("T", 1, sink, None, 0, false, false, None, Some(true), None); assert!( modes_syncs(&seen).is_empty(), "no attachRequestId => no sync" @@ -5768,6 +7464,7 @@ mod tests { false, None, Some(true), + None, ); assert!(outcome.found); @@ -5843,7 +7540,18 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink, _seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); + let outcome = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); assert!(outcome.found); reg.set_auto_kill_idle_minutes(1); // Far past any threshold, but a client is attached -- legacy: @@ -6020,7 +7728,18 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink, _seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); + let outcome = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); assert!(outcome.found); reg.set_auto_kill_idle_minutes(1); reg.backdate_last_activity("T", now_ms() - 10 * 60_000); @@ -6047,7 +7766,18 @@ mod tests { let reg = TerminalRegistry::new(); reg.insert_headless("T", "S"); let (sink, _seen) = collector(); - let outcome = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); + let outcome = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); assert!(outcome.found); // A second, already-detached terminal whose countdown must NOT be // disturbed by conn 1's disconnect. @@ -6078,8 +7808,19 @@ mod tests { reg.insert_headless("T", "S"); let (sink, _seen) = collector(); assert!( - reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None) - .found + reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None + ) + .found ); reg.set_auto_kill_idle_minutes(15); // the shipped default reg.remove_connection(1); // socket drop, NOT an explicit detach @@ -6104,8 +7845,19 @@ mod tests { reg.insert_headless("T", "S"); let (sink, _seen) = collector(); assert!( - reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None) - .found + reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None + ) + .found ); reg.set_auto_kill_idle_minutes(15); reg.remove_connection(1); @@ -6124,14 +7876,36 @@ mod tests { reg.insert_headless("T", "S"); let (sink, _seen) = collector(); assert!( - reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None) - .found + reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None + ) + .found ); reg.detach("T", 1); // explicitly released — fast-reap eligible let (sink2, _seen2) = collector(); assert!( - reg.attach("T", 2, sink2, Some("b".into()), 0, false, false, None, None) - .found + reg.attach( + "T", + 2, + sink2, + Some("b".into()), + 0, + false, + false, + None, + None, + None + ) + .found ); reg.set_auto_kill_idle_minutes(15); reg.remove_connection(2); // wanted again, then socket drop @@ -6222,7 +7996,18 @@ mod tests { reg.feed("T", frame(2, "abcdefghij", "S")); // another 10 bytes -> over cap let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); + let _ = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); let replayed = outputs(&seen); // Whole-frame FIFO eviction keeps at least one frame; the FIRST frame // must have been evicted once the second pushed bytes over the cap. @@ -6240,7 +8025,18 @@ mod tests { reg.feed("T", frame(2, "abcdefghij", "S")); let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("a".into()), 0, false, false, None, None); + let _ = reg.attach( + "T", + 1, + sink, + Some("a".into()), + 0, + false, + false, + None, + None, + None, + ); let replayed = outputs(&seen); assert_eq!( replayed.len(), @@ -6282,6 +8078,7 @@ mod tests { false, None, None, + None, ); let ascii_chars: usize = outputs(&seen_a) .iter() @@ -6311,6 +8108,7 @@ mod tests { false, None, None, + None, ); let box_chars: usize = outputs(&seen_b) .iter() @@ -6571,7 +8369,18 @@ mod tests { } let (sink, seen) = collector(); - let _ = reg.attach("T", 1, sink, Some("r".into()), 0, false, false, None, None); + let _ = reg.attach( + "T", + 1, + sink, + Some("r".into()), + 0, + false, + false, + None, + None, + None, + ); let retained_chars: usize = outputs(&seen).iter().map(|f| f.data.chars().count()).sum(); assert!( retained_chars as i64 <= cap, diff --git a/crates/freshell-ws/src/connection_writer.rs b/crates/freshell-ws/src/connection_writer.rs index 3a73bf64f..20c6f1562 100644 --- a/crates/freshell-ws/src/connection_writer.rs +++ b/crates/freshell-ws/src/connection_writer.rs @@ -292,7 +292,11 @@ impl WriterSender { return false; } }; - if meta.is_none() && !exit { + // Restore contract: a directly pushed `terminal.output.gap` (the paced + // replay core's retention gaps) joins the OUTPUT queue as a sequenced + // control like `terminal.exit` — see the gap arm below. + let sequenced_gap = matches!(&msg, ServerMessage::TerminalOutputGap(_)); + if meta.is_none() && !exit && !sequenced_gap { return self .push_control(Message::Text(json.into()), None, supersedes.as_deref()) .is_ok(); @@ -328,11 +332,8 @@ impl WriterSender { self.fail(WriterExit::OutputCapacityExceeded); return false; } - } else { + } else if let ServerMessage::TerminalExit(exit) = &msg { // Preserve final-output -> exit. It must not use the control lane. - let ServerMessage::TerminalExit(exit) = msg else { - unreachable!("sequenced exit only") - }; let priority = queues.interest.priority(&exit.terminal_id); // Sequenced exits are zero-weight, exactly as legacy queued them: // they can never force an eviction nor close the connection, and @@ -356,6 +357,36 @@ impl WriterSender { } // A dead terminal never needs its attach fallback again. queues.interest.detach(&exit.terminal_id); + } else { + // Restore contract (responsive-terminal-restore): a + // `terminal.output.gap` pushed DIRECTLY by the paced replay core + // (retention loss at attach / mid-replay expiry) is sequenced + // WITH the terminal's output — exactly the `terminal.exit` + // zero-weight non-evictable control treatment. The control lane + // would preempt it AHEAD of already-admitted pages, breaking + // per-terminal sequence order (the queue's own gap markers, by + // contrast, materialize at lease time from eviction and never + // pass through here). + let ServerMessage::TerminalOutputGap(gap) = &msg else { + unreachable!("meta-less output frames are exit or gap only") + }; + let priority = queues.interest.priority(&gap.terminal_id); + if queues + .output + .push( + &gap.terminal_id, + priority, + Message::Text(json.into()), + 0, + None, + seq, + ) + .is_err() + { + drop(queues); + self.fail(WriterExit::OutputCapacityExceeded); + return false; + } } drop(queues); self.shared.ready.notify_one(); diff --git a/crates/freshell-ws/src/connection_writer_tests.rs b/crates/freshell-ws/src/connection_writer_tests.rs index cbbe6fb69..0e1bc5291 100644 --- a/crates/freshell-ws/src/connection_writer_tests.rs +++ b/crates/freshell-ws/src/connection_writer_tests.rs @@ -639,3 +639,117 @@ async fn queue_overflow_gap_without_paced_negotiation_keeps_the_frozen_shape() { ); pump.finish_frame(next.output_bytes, next.control_bytes); } + +/// A directly pushed `terminal.output.gap` (the paced path emits retention +/// gaps through `push_server`, unlike the queue's own lease-time gap +/// materialization): the gap is sequenced WITH the terminal's output — it +/// must lease strictly AFTER output admitted before it, never jumping ahead +/// on the preemptive control lane, and it must never be evictable or +/// byte-charged. +#[tokio::test] +async fn server_pushed_restore_gap_stays_ordered_with_the_terminals_output() { + let (sender, pump) = WriterSender::new(100_000, 4096, Duration::from_secs(10)); + assert!(sender.push_server(output(1))); + let gap = ServerMessage::TerminalOutputGap(freshell_protocol::TerminalOutputGap { + terminal_id: "term".into(), + stream_id: "stream".into(), + attach_request_id: Some("attach".into()), + from_seq: 1, + to_seq: 9, + reason: freshell_protocol::TerminalOutputGapReason::ReplayWindowExceeded, + head_seq: Some(12), + oldest_retained_seq: Some(10), + }); + assert!(sender.push_server(gap)); + + let first = pump.take_next().unwrap().unwrap(); + assert!( + leased_text(&first.frame).contains("data-1"), + "output admitted BEFORE the gap leases first" + ); + pump.finish_frame(first.output_bytes, first.control_bytes); + + let second = pump.take_next().unwrap().unwrap(); + let gap_json: serde_json::Value = serde_json::from_str(&leased_text(&second.frame)).unwrap(); + assert_eq!(gap_json["type"], "terminal.output.gap"); + assert_eq!( + second.output_bytes, 0, + "the gap leases as a zero-weight sequenced control (never byte-charged)" + ); + pump.finish_frame(second.output_bytes, second.control_bytes); +} + +/// The server-pushed restore gap must survive queue overflow eviction: like +/// `terminal.exit`, it is a non-evictable sequenced control — the byte cap +/// evicts payload frames, never the gap. +#[tokio::test] +async fn server_pushed_restore_gap_is_not_evictable_under_overflow() { + let (sender, pump) = overflow_writer(); + assert!(sender.push_server(output(1))); + let gap = ServerMessage::TerminalOutputGap(freshell_protocol::TerminalOutputGap { + terminal_id: "term".into(), + stream_id: "stream".into(), + attach_request_id: Some("attach".into()), + from_seq: 2, + to_seq: 2, + reason: freshell_protocol::TerminalOutputGapReason::ReplayWindowExceeded, + head_seq: None, + oldest_retained_seq: None, + }); + assert!(sender.push_server(gap)); + // Overflow: output(2) does not fit and evicts the OLDEST EVICTABLE entry — + // output(1) — never the non-evictable gap. + assert!(sender.push_server(output(2))); + + let first = pump.take_next().unwrap().unwrap(); + let first_json: serde_json::Value = serde_json::from_str(&leased_text(&first.frame)).unwrap(); + assert_eq!( + first_json["type"], "terminal.output.gap", + "the queue-overflow gap head leases first (the evicted output(1))" + ); + pump.finish_frame(first.output_bytes, first.control_bytes); + + let second = pump.take_next().unwrap().unwrap(); + let second_json: serde_json::Value = serde_json::from_str(&leased_text(&second.frame)).unwrap(); + assert_eq!( + second_json["type"], "terminal.output.gap", + "the server-pushed restore gap survives the overflow eviction" + ); + assert_eq!(second_json["fromSeq"], 2); + pump.finish_frame(second.output_bytes, second.control_bytes); + + let third = pump.take_next().unwrap().unwrap(); + assert!(leased_text(&third.frame).contains("data-2")); + pump.finish_frame(third.output_bytes, third.control_bytes); +} + +/// Task-2 review follow-up (Minor 2): a writer stop that lands while the Gap +/// arm has RELEASED the admission lock to resolve negotiated bounds (after +/// the pop, before the materialize) must yield NO flushed frame — the +/// re-acquire re-check of `closed` aborts the lease and the pump exits via +/// the stop watch, leaving the popped gap unflushed. +#[tokio::test] +async fn writer_stop_landing_during_gap_bounds_resolution_flushes_no_frame() { + let (sender, pump) = overflow_writer(); + let stopping = sender.clone(); + sender.set_paced_replay_gap_bounds(Arc::new(move |_| { + // The stop lands mid-resolution: the gap was already popped and the + // admission lock released. + stopping.stop_without_close(); + Some(freshell_terminal::ReplayBounds { + head_seq: 9, + oldest_retained_seq: 2, + }) + })); + assert!(sender.push_server(output(1))); + assert!(sender.push_server(output(2))); // evicts output(1) -> queue gap + + let capture = Arc::new(Capture::default()); + let task = tokio::spawn(pump.run(TestSink(Arc::clone(&capture)))); + assert_eq!(join(task).await, WriterExit::Stopped); + assert!( + text_frames(&capture).is_empty(), + "a stop before the lease must leave nothing flushed" + ); + assert_eq!(sender.pending_output_bytes(), 0); +} diff --git a/crates/freshell-ws/src/lib.rs b/crates/freshell-ws/src/lib.rs index f5ab1fa6e..f4a64445d 100644 --- a/crates/freshell-ws/src/lib.rs +++ b/crates/freshell-ws/src/lib.rs @@ -56,6 +56,7 @@ pub mod opencode_association; pub mod opencode_lane; pub mod opencode_signal; pub mod origin; +pub(crate) mod paced_replay; pub mod pane_identity_binder; pub mod pane_ledger; pub mod reconcile; diff --git a/crates/freshell-ws/src/paced_replay.rs b/crates/freshell-ws/src/paced_replay.rs new file mode 100644 index 000000000..b4b275949 --- /dev/null +++ b/crates/freshell-ws/src/paced_replay.rs @@ -0,0 +1,475 @@ +//! Responsive-terminal-restore Workstream 1 — the connection-side pacing +//! coordinator for negotiated (`pacedTerminalReplayV1`) terminal replay. +//! +//! The registry owns the page reads ([`freshell_terminal::TerminalRegistry`] +//! `attach`'s paced start, `next_replay_page`, `next_paced_tail_page`) and +//! the per-subscriber deferral; THIS module owns the session state the wire +//! protocol needs: the fixed catch-up target, the production cursor, the +//! credited-consumed window that gates continuation credits, and the +//! drive loop that turns a valid [`TerminalReplayCredit`] into the next +//! page (or the negotiated retention gap + continuation, or the tail drain +//! that completes the session). +//! +//! Credit rules (all under the connection's negotiated capability; ignored +//! otherwise): a credit whose `attachRequestId` does not match the ACTIVE +//! session is a stale generation (superseded or completed — ignored); a +//! `consumedSeq` outside `(credited, page_end]` is out of window (ignored — +//! no double-grant, no phantom grant past the last-sent page). A valid +//! credit produces exactly ONE further replay page, so at most ONE +//! unacknowledged page per (connection, terminal) exists at any time. Gap +//! rounds continue within the same credit until a frame-carrying page is +//! produced — an accepted credit always makes byte progress, never a +//! zero-progress demand for more credit. +//! +//! When the cursor reaches the target, the accumulated live range +//! `(target, head]` drains as ordinary delivery (pages, not credit-gated) +//! and the registry clears the deferral ATOMICALLY when the ring is +//! drained — live output can never overtake an un-sent page. + +use std::collections::HashMap; + +use freshell_protocol::{ + ServerMessage, TerminalOutputGap, TerminalOutputGapReason, TerminalReplayCredit, +}; +use freshell_terminal::{FrameSink, PacedPage, PacedSessionDesc, PacedTailPage, TerminalRegistry}; + +/// One active paced replay session for a (connection, terminal). Owned by +/// the connection's dispatch loop; a re-attach replaces it, a detach or +/// socket drop discards it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PacedSession { + pub terminal_id: String, + pub stream_id: String, + /// The attach generation this session serves — the stale-credit key. + pub attach_request_id: String, + /// FIXED catch-up target: the head at attach time. Ongoing output never + /// extends it (frames past the target are tail delivery). + pub target: i64, + /// The retention-adjusted baseline the session started from. + pub effective_since: i64, + /// Highest credited consumed seq (starts at the baseline; the + /// double-grant guard). + pub credited: i64, + /// Production cursor: the last seq sent in a page. + pub page_end: i64, + /// Pages produced so far (observability). + pub pages: u64, +} + +impl PacedSession { + /// Adopt the registry's attach-time session description (the first page + /// was produced under the attach lock and is sunk by the caller). + pub(crate) fn from_desc(desc: PacedSessionDesc) -> Self { + let started = desc.page_end > desc.effective_since; + Self { + terminal_id: desc.terminal_id, + stream_id: desc.stream_id, + attach_request_id: desc.attach_request_id, + target: desc.target, + effective_since: desc.effective_since, + credited: desc.effective_since, + page_end: desc.page_end, + pages: u64::from(started), + } + } +} + +/// The per-connection session table (at most one session per terminal). +#[derive(Default)] +pub(crate) struct PacedSessions { + inner: HashMap, +} + +impl PacedSessions { + /// Install (or replace — the re-attach supersede) a session. + pub(crate) fn insert(&mut self, session: PacedSession) { + self.inner.insert(session.terminal_id.clone(), session); + } + + pub(crate) fn get_mut(&mut self, terminal_id: &str) -> Option<&mut PacedSession> { + self.inner.get_mut(terminal_id) + } + + /// Cancel one terminal's session (`terminal.detach`). + pub(crate) fn cancel(&mut self, terminal_id: &str) { + self.inner.remove(terminal_id); + } + + pub(crate) fn remove(&mut self, terminal_id: &str) -> Option { + self.inner.remove(terminal_id) + } +} + +/// The verdict for one inbound credit (observability's status field). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CreditVerdict { + Accepted, + StaleGeneration, + BeyondWindow, + NonNegotiated, +} + +impl CreditVerdict { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Accepted => "accepted", + Self::StaleGeneration => "stale_generation", + Self::BeyondWindow => "beyond_window", + Self::NonNegotiated => "non_negotiated", + } + } +} + +/// Validate one credit against the session and, when valid, advance the +/// credited cursor (the grant). Pure — the drive loop does the producing. +pub(crate) fn validate_credit( + session: &mut PacedSession, + credit: &TerminalReplayCredit, +) -> CreditVerdict { + if session.attach_request_id != credit.attach_request_id { + return CreditVerdict::StaleGeneration; + } + if credit.consumed_seq <= session.credited || credit.consumed_seq > session.page_end { + return CreditVerdict::BeyondWindow; + } + session.credited = credit.consumed_seq; + CreditVerdict::Accepted +} + +/// The negotiated retention gap for an `Expired` read: the exact lost +/// interval plus the task-2 bounds fields, stamped with the session's +/// generation. +fn retention_gap( + session: &PacedSession, + lost_from: i64, + lost_to: i64, + head_seq: i64, + oldest_retained_seq: i64, +) -> ServerMessage { + ServerMessage::TerminalOutputGap(TerminalOutputGap { + terminal_id: session.terminal_id.clone(), + stream_id: session.stream_id.clone(), + attach_request_id: Some(session.attach_request_id.clone()), + from_seq: lost_from, + to_seq: lost_to, + reason: TerminalOutputGapReason::ReplayWindowExceeded, + head_seq: Some(head_seq), + oldest_retained_seq: Some(oldest_retained_seq), + }) +} + +/// The outcome of driving a session one round (attach start or one valid +/// credit). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DriveOutcome { + /// A replay page is outstanding (uncredited); the session waits for the + /// next credit. + Active, + /// The session completed: the tail drained and the registry cleared the + /// deferral (the `ws.restore.paced_complete` event is emitted here). + Completed, + /// The terminal (or the connection's subscriber) disappeared — the + /// session is cancelled; the registry has no deferral left to clear. + Gone, +} + +/// Produce the next page for a session after a valid credit (or drain the +/// tail when the cursor reached the target). Exactly one replay page per +/// credit; `Expired` rounds emit the negotiated retention gap and continue +/// within the same credit until a frame-carrying page exists. The tail +/// drains without credit and completes the session when the registry's +/// atomic clear fires (`CaughtUp`). +pub(crate) fn drive_session( + registry: &TerminalRegistry, + conn_id: u64, + sink: &FrameSink, + session: &mut PacedSession, + budget: i64, +) -> DriveOutcome { + // Replay phase: page (target, ...] toward the fixed target. + if session.page_end < session.target { + loop { + match registry.next_replay_page( + &session.terminal_id, + conn_id, + session.page_end, + session.target, + budget, + ) { + PacedPage::Frames { + messages, end_seq, .. + } => { + for message in messages { + sink(message); + } + session.page_end = end_seq; + session.pages += 1; + if session.page_end < session.target { + return DriveOutcome::Active; + } + break; // cursor reached the target -> tail + } + PacedPage::Done => break, + PacedPage::Expired { + lost_from, + lost_to, + resume_from, + head_seq, + oldest_retained_seq, + } => { + sink(retention_gap( + session, + lost_from, + lost_to, + head_seq, + oldest_retained_seq, + )); + tracing::info!( + terminal_id = %session.terminal_id, + lost_from, + lost_to, + resume_from, + "ws.restore.paced_expired" + ); + session.page_end = resume_from; + continue; + } + PacedPage::Gone => { + tracing::warn!( + terminal_id = %session.terminal_id, + attach_request_id = %session.attach_request_id, + "ws.restore.paced_gone" + ); + return DriveOutcome::Gone; + } + } + } + } + // Tail phase: ordinary delivery of the accumulated live range; the + // registry clears the deferral atomically when the ring is drained. + loop { + match registry.next_paced_tail_page(&session.terminal_id, conn_id, session.page_end, budget) + { + PacedTailPage::Frames { + messages, end_seq, .. + } => { + for message in messages { + sink(message); + } + session.page_end = end_seq; + session.pages += 1; + } + PacedTailPage::Expired { + lost_from, + lost_to, + resume_from, + head_seq, + oldest_retained_seq, + } => { + sink(retention_gap( + session, + lost_from, + lost_to, + head_seq, + oldest_retained_seq, + )); + tracing::info!( + terminal_id = %session.terminal_id, + lost_from, + lost_to, + resume_from, + "ws.restore.paced_expired" + ); + session.page_end = resume_from; + } + PacedTailPage::CaughtUp => { + tracing::info!( + terminal_id = %session.terminal_id, + attach_request_id = %session.attach_request_id, + last_seq = session.page_end, + pages = session.pages, + "ws.restore.paced_complete" + ); + return DriveOutcome::Completed; + } + PacedTailPage::Gone => { + tracing::warn!( + terminal_id = %session.terminal_id, + attach_request_id = %session.attach_request_id, + "ws.restore.paced_gone" + ); + return DriveOutcome::Gone; + } + } + } +} + +/// Begin one negotiated session after a paced attach: sink the first page +/// (produced under the attach lock; sunk now that it is released), emit +/// `ws.restore.paced_start`, and — ONLY when the first page already reached +/// the target — run the initial tail drain (an attach with a short or empty +/// replay completes without ever needing a credit). A first page that is a +/// bounded prefix leaves the session ACTIVE with exactly ONE outstanding +/// page: the next page is produced on the first credit, never before. +pub(crate) fn start_session( + registry: &TerminalRegistry, + conn_id: u64, + sink: &FrameSink, + sessions: &mut PacedSessions, + start: freshell_terminal::PacedAttachStart, + requested_since_seq: i64, + max_replay_bytes: Option, +) { + let page_bytes = start.session.page_bytes; + let mut session = PacedSession::from_desc(start.session); + for message in start.first_page { + sink(message); + } + tracing::info!( + terminal_id = %session.terminal_id, + attach_request_id = %session.attach_request_id, + requested_since = requested_since_seq, + effective_since = session.effective_since, + target = session.target, + page_bytes, + max_replay_bytes = ?max_replay_bytes, + "ws.restore.paced_start" + ); + if session.page_end >= session.target { + let budget = registry.paced_page_max_bytes(); + match drive_session(registry, conn_id, sink, &mut session, budget) { + // The tail drain never returns Active — but if it somehow did, + // keeping the session installed is the safe arm (the next credit + // drives it) rather than dropping live-ordering state. + DriveOutcome::Active => sessions.insert(session), + DriveOutcome::Completed | DriveOutcome::Gone => {} + } + } else { + // The first page is the one outstanding, uncredited page. + sessions.insert(session); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn session_fixture() -> PacedSession { + PacedSession { + terminal_id: "T".into(), + stream_id: "S".into(), + attach_request_id: "arid-1".into(), + target: 100, + effective_since: 10, + credited: 10, + page_end: 50, + pages: 1, + } + } + + fn credit(arid: &str, consumed_seq: i64) -> TerminalReplayCredit { + TerminalReplayCredit { + terminal_id: "T".into(), + stream_id: "S".into(), + attach_request_id: arid.into(), + consumed_seq, + } + } + + #[test] + fn valid_credit_advances_the_credited_cursor() { + let mut session = session_fixture(); + assert_eq!( + validate_credit(&mut session, &credit("arid-1", 50)), + CreditVerdict::Accepted + ); + assert_eq!(session.credited, 50); + // The production position stays at the last-sent page end. + assert_eq!(session.page_end, 50); + } + + #[test] + fn stale_generation_credit_is_ignored() { + let mut session = session_fixture(); + assert_eq!( + validate_credit(&mut session, &credit("arid-old", 50)), + CreditVerdict::StaleGeneration + ); + assert_eq!(session.credited, 10, "a stale credit grants nothing"); + } + + #[test] + fn duplicate_and_beyond_window_credits_are_ignored() { + let mut session = session_fixture(); + // Beyond the last-sent page's end. + assert_eq!( + validate_credit(&mut session, &credit("arid-1", 51)), + CreditVerdict::BeyondWindow + ); + // At-or-below the already-credited cursor (double grant). + assert_eq!( + validate_credit(&mut session, &credit("arid-1", 10)), + CreditVerdict::BeyondWindow + ); + assert_eq!(session.credited, 10); + // A VALID credit still works after the ignored ones. + assert_eq!( + validate_credit(&mut session, &credit("arid-1", 50)), + CreditVerdict::Accepted + ); + } + + #[test] + fn partial_consumption_within_the_page_window_grants_once() { + let mut session = session_fixture(); + // consumedSeq inside (credited, page_end] — valid; production + // continues from the last-sent end, so no frame is ever re-sent. + assert_eq!( + validate_credit(&mut session, &credit("arid-1", 30)), + CreditVerdict::Accepted + ); + assert_eq!(session.credited, 30); + assert_eq!( + validate_credit(&mut session, &credit("arid-1", 30)), + CreditVerdict::BeyondWindow, + "the same value cannot grant twice" + ); + assert_eq!( + validate_credit(&mut session, &credit("arid-1", 50)), + CreditVerdict::Accepted + ); + } + + #[test] + fn session_adoption_counts_the_first_page() { + let desc = PacedSessionDesc { + terminal_id: "T".into(), + stream_id: "S".into(), + attach_request_id: "a".into(), + target: 9, + effective_since: 3, + page_end: 7, + page_bytes: 123, + }; + let session = PacedSession::from_desc(desc); + assert_eq!(session.credited, 3, "crediting starts at the baseline"); + assert_eq!(session.page_end, 7); + assert_eq!(session.pages, 1); + + // An attach with nothing to replay starts with no pages. + let empty = PacedSessionDesc { + page_end: 3, + effective_since: 3, + ..PacedSessionDesc { + terminal_id: "T".into(), + stream_id: "S".into(), + attach_request_id: "a".into(), + target: 3, + effective_since: 3, + page_end: 3, + page_bytes: 0, + } + }; + assert_eq!(PacedSession::from_desc(empty).pages, 0); + } +} diff --git a/crates/freshell-ws/src/terminal.rs b/crates/freshell-ws/src/terminal.rs index fc094bb51..6d0e9f8ef 100644 --- a/crates/freshell-ws/src/terminal.rs +++ b/crates/freshell-ws/src/terminal.rs @@ -387,6 +387,11 @@ async fn run_loop( let mut writer_task = tokio::spawn(writer.run(socket_tx).instrument(tracing::Span::current())); let _writer_lifetime = connection_writer::AbortWriterOnDrop(writer_task.abort_handle()); let mut writer_finished = false; + // Responsive-terminal-restore W1: this connection's paced replay + // sessions. Owned by the loop — a socket drop discards them (the + // registry-side deferrals die with the subscribers remove_connection + // sweeps), a detach cancels one, a re-attach replaces it. + let mut paced_sessions = crate::paced_replay::PacedSessions::default(); let conn_sink: FrameSink = { let sender = ws_tx.clone(); Arc::new(move |msg| { @@ -530,6 +535,7 @@ async fn run_loop( &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut paced_sessions, ) .await { @@ -778,6 +784,9 @@ async fn handle_client_text( // D8: the connection's hello-stamped client identity (refreshed by // `tabs.sync.push` below) — the provenance source for ledger stamps. conn_identity: &mut ConnectionIdentity, + // Responsive-terminal-restore W1: this connection's paced replay + // sessions (one per attached terminal; the pacing coordinator's state). + paced_sessions: &mut crate::paced_replay::PacedSessions, ) -> bool { // Accept-and-strip: unknown/unparseable frames are ignored (matches the // runtime's tolerance; the handshake already gated auth). @@ -1477,6 +1486,9 @@ async fn handle_client_text( // session lost before its first snapshot. let asserted_at = now_ms(); maybe_restamp_on_attach(&attach, state, conn_identity, asserted_at).await; + // Observability inputs captured before the attach moves. + let requested_since_seq = attach.since_seq.unwrap_or(0); + let attach_max_replay_bytes = attach.max_replay_bytes; let attached = match handle_attach( attach, state, @@ -1485,8 +1497,25 @@ async fn handle_client_text( terminal_output_batch_v1, paced_terminal_replay_v1, ) { - Some(err) => send(ws_tx, &err).await, - None => true, + AttachReply::Error(err) => send(ws_tx, &err).await, + AttachReply::Legacy => true, + AttachReply::Paced(start) => { + // The paced replay core (responsive-terminal-restore + // W1): sink the first page (the registry produced it + // under the attach lock), emit the session start, and + // drain to quiet — a short replay can complete here + // without ever needing a credit. + crate::paced_replay::start_session( + &state.registry, + conn_id, + conn_sink, + paced_sessions, + *start, + requested_since_seq, + attach_max_replay_bytes, + ); + true + } }; // The window closes with the guard (exactly-once). drop(attach_guard); @@ -1562,8 +1591,29 @@ async fn handle_client_text( } } ClientMessage::TerminalDetach(detach) => { + // Responsive-terminal-restore: an explicit detach cancels the + // connection's paced session for this terminal (the registry-side + // deferral dies with the subscriber the detach removes). + paced_sessions.cancel(&detach.terminal_id); handle_detach(&detach, ws_tx, state, conn_id).await } + // Responsive-terminal-restore Workstream 1: the paced replay + // continuation credit. All rules sit under the connection's + // negotiated capability — a non-negotiated connection's credits are + // inert (logged, then ignored). + ClientMessage::TerminalReplayCredit(replay_credit) => { + if !paced_terminal_replay_v1 { + tracing::info!( + terminal_id = %replay_credit.terminal_id, + consumed_seq = replay_credit.consumed_seq, + status = crate::paced_replay::CreditVerdict::NonNegotiated.as_str(), + "ws.restore.credit" + ); + true + } else { + handle_replay_credit(&replay_credit, state, conn_id, conn_sink, paced_sessions) + } + } ClientMessage::TerminalKill(kill) => { // b8ke ext r24 F2: the kill's coordinator transitions record // the connection's real device/client identity as the @@ -6926,13 +6976,32 @@ async fn maybe_restamp_on_attach( } /// `terminal.attach` — resolve the terminal in the shared registry and attach THIS -/// connection to it: the registry enqueues `terminal.attach.ready`, replays the -/// scrollback (seq-ordered, stamped with this attach's id + `source:'replay'`), and -/// registers the connection so live output fans out — all onto `conn_sink`, which -/// the select loop drains to the socket. Attaching to an unknown terminal returns -/// the reference's `error{INVALID_TERMINAL_ID, "Terminal not running"}` frame for -/// the caller to send (`ws-handler.ts:2730-2735`; restored by kata dtfn — the SPA's -/// recovery ladder recreates the pane). `None` = attached. +/// connection to it: the registry enqueues `terminal.attach.ready` and replays the +/// scrollback (seq-ordered, stamped with this attach's id + `source:'replay'`) onto +/// `conn_sink`, which the select loop drains to the socket. Attaching to an +/// unknown terminal returns the reference's `error{INVALID_TERMINAL_ID, +/// "Terminal not running"}` frame for the caller to send +/// (`ws-handler.ts:2730-2735`; restored by kata dtfn — the SPA's recovery ladder +/// recreates the pane). `Ok(None)` = attached with no reply. +/// +/// Responsive-terminal-restore Workstream 1: a NEGOTIATED +/// (`pacedTerminalReplayV1`) attach to a Running terminal with an +/// attachRequestId returns the paced session start instead — the registry +/// armed the subscriber's deferral and produced the FIRST page under the +/// attach lock; the caller sinks that page after the lock is released and +/// owns the pacing session (credits, tail drain, completion). +/// +/// TERM-07: the attach's `maxReplayBytes` threads through BOTH +/// geometry-authorized and geometry-skipped paths into the registry call, +/// where it is recorded on the subscriber with no delivery-behavior change +/// (the paced-start observability event reports it; the increment-3 +/// snapshot work consumes it). +enum AttachReply { + Legacy, + Error(Box), + Paced(Box), +} + fn handle_attach( attach: TerminalAttach, state: &WsState, @@ -6940,10 +7009,10 @@ fn handle_attach( conn_sink: &FrameSink, terminal_output_batch_v1: bool, paced_terminal_replay_v1: bool, -) -> Option { +) -> AttachReply { // STATE-SYNC FIX 1 increment 2a: stamp the canonical identity onto // `attach.ready` from the shared identity registry (create-time - // resume ids AND locator-associated ids both live there); the + // resume ids AND locator-associated ids both live here); the // registry crate is identity-agnostic, so it's resolved here. let canonical_session_ref = state.identity.session_ref_for(&attach.terminal_id); @@ -6957,13 +7026,15 @@ fn handle_attach( attach.expected_session_ref.as_ref(), canonical_session_ref.as_ref(), ); + // TERM-07 seam: the client's replay-budget request rides both paths. + let max_replay_bytes = attach.max_replay_bytes; let outcome = if geometry_identity_ok { let cols = attach.cols.clamp(0, u16::MAX as i64) as u16; let rows = attach.rows.clamp(0, u16::MAX as i64) as u16; - // `paced_terminal_replay_v1` parks the negotiated capability on the - // attach's subscriber (alongside `terminal_output_batch_v1`); - // Workstream 1's paced replay core (registry pages + coordinator, - // task 3) consumes it to gate paced restore delivery. + // `paced_terminal_replay_v1` gates the paced replay core + // (responsive-terminal-restore): a negotiated attach with an + // attachRequestId to a Running terminal returns the paced session + // start; every other shape keeps the legacy inline replay. state.registry.attach_with_geometry( &attach.terminal_id, conn_id, @@ -6977,6 +7048,7 @@ fn handle_attach( // (xterm recreation / user reset). Forwards the wire field 1:1; the // registry owns the emit-vs-skip gating. attach.surface_reset, + max_replay_bytes, attach.intent, cols, rows, @@ -6992,10 +7064,14 @@ fn handle_attach( paced_terminal_replay_v1, canonical_session_ref, attach.surface_reset, + max_replay_bytes, ) }; if outcome.found { - return None; + return match outcome.paced { + Some(start) => AttachReply::Paced(Box::new(start)), + None => AttachReply::Legacy, + }; } // Kata dtfn: `AttachOutcome{found:false}` was silently discarded here, // wedging any attach against an unknown id (stale pre-restart id, typo'd @@ -7004,7 +7080,7 @@ fn handle_attach( // gate accepts it (attachRequestIds live in the `pane:N:nanoid` namespace, // never colliding with createRequestIds — see ws-client's // clearTrackedCreate-on-error behavior). - Some(ServerMessage::Error(ErrorMsg { + AttachReply::Error(Box::new(ServerMessage::Error(ErrorMsg { owner_kind: None, owner_generation: None, owner_epoch: None, @@ -7018,7 +7094,52 @@ fn handle_attach( terminal_id: Some(attach.terminal_id), terminal_exit_code: None, live_terminal_id: None, - })) + }))) +} + +/// One `terminal.replay.credit` on a negotiated connection +/// (responsive-terminal-restore Workstream 1): validate against the active +/// session's generation + outstanding-page window, and on acceptance +/// produce the next page (or the negotiated retention gap + continuation, +/// or — once the cursor reaches the target — the tail drain that completes +/// the session). Every non-accepted verdict is inert (observed via +/// `ws.restore.credit`, never a client-visible error). +fn handle_replay_credit( + replay_credit: &freshell_protocol::TerminalReplayCredit, + state: &WsState, + conn_id: u64, + conn_sink: &FrameSink, + paced_sessions: &mut crate::paced_replay::PacedSessions, +) -> bool { + use crate::paced_replay::DriveOutcome; + // Identifiers/measurements only, per the restore observability contract. + let observe = |verdict: crate::paced_replay::CreditVerdict| { + tracing::info!( + terminal_id = %replay_credit.terminal_id, + consumed_seq = replay_credit.consumed_seq, + status = verdict.as_str(), + "ws.restore.credit" + ); + }; + let Some(session) = paced_sessions.get_mut(&replay_credit.terminal_id) else { + // No active session accepts this credit (completed, detached, or a + // terminal never paced): a stale generation. + observe(crate::paced_replay::CreditVerdict::StaleGeneration); + return true; + }; + let verdict = crate::paced_replay::validate_credit(session, replay_credit); + observe(verdict); + if verdict != crate::paced_replay::CreditVerdict::Accepted { + return true; + } + let budget = state.registry.paced_page_max_bytes(); + match crate::paced_replay::drive_session(&state.registry, conn_id, conn_sink, session, budget) { + DriveOutcome::Active => {} + DriveOutcome::Completed | DriveOutcome::Gone => { + paced_sessions.remove(&replay_credit.terminal_id); + } + } + true } /// Node's `resizeIfSessionMatches` identity guard @@ -10357,6 +10478,7 @@ mod pane_reconcile_gate_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!( @@ -10382,6 +10504,7 @@ mod pane_reconcile_gate_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(pong_ok); @@ -10427,6 +10550,7 @@ mod pane_reconcile_gate_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(ok); @@ -10452,6 +10576,7 @@ mod pane_reconcile_gate_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(ok, "attempt {attempt}: a full queue must be answered"); @@ -10794,6 +10919,7 @@ mod host_stats_dispatch_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(ok); @@ -10826,6 +10952,7 @@ mod host_stats_dispatch_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(ok); @@ -10848,6 +10975,7 @@ mod host_stats_dispatch_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(pong_ok); @@ -10891,6 +11019,7 @@ mod host_stats_dispatch_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(ok); @@ -10919,6 +11048,7 @@ mod host_stats_dispatch_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(ok); @@ -10965,6 +11095,7 @@ mod host_stats_dispatch_tests { &create_cancel_rx, &mut host_stats_last_refresh_at, &mut conn_identity, + &mut Default::default(), ) .await; assert!(ok); diff --git a/crates/freshell-ws/tests/paced_replay.rs b/crates/freshell-ws/tests/paced_replay.rs new file mode 100644 index 000000000..a597d198f --- /dev/null +++ b/crates/freshell-ws/tests/paced_replay.rs @@ -0,0 +1,860 @@ +//! Responsive-terminal-restore Workstream 1 — the paced replay core, +//! end-to-end on REAL sockets: a REAL axum server (ephemeral loopback +//! port), a REAL PTY (`TerminalRegistry`), and REAL tokio-tungstenite WS +//! clients, one of which negotiates `pacedTerminalReplayV1` and drives +//! continuation credits over the raw socket. +//! +//! The harness follows `hello_capabilities.rs` (the `connect_async` +//! `WsClient` type — the two files' WS client types are deliberately NOT +//! shared). The page budget and the scrollback ring are per-server registry +//! knobs, so fixtures are deterministic without env races. +//! +//! Core contract under test: a negotiated attach gets `attach.ready` bounds +//! plus ONE bounded first page; further replay pages flow only on valid +//! `terminal.replay.credit` (stale generations, out-of-window values, and +//! credits from non-negotiated connections are ignored); live output +//! produced during the replay is delivered strictly AFTER the pages covering +//! it, in seq order; retention expiry mid-replay reports the exact lost +//! interval with the task-2 bounds fields and continues from the new +//! baseline; a re-attach supersedes the old session; a disconnect +//! mid-replay leaves the terminal running and re-attachable. + +use std::sync::Arc; +use std::time::Duration; + +use futures_util::{SinkExt, StreamExt}; +use tokio::net::TcpListener; +use tokio_tungstenite::tungstenite::Message as WsMessage; + +use freshell_ws::WsState; + +const AUTH_TOKEN: &str = "s3cr3t-token-abcdef"; +/// Deliberately small page budget: deterministic multi-page fixtures with +/// small floods (the production default is 128 KiB). +const PAGE_BUDGET: i64 = 4096; + +fn test_settings_value() -> serde_json::Value { + serde_json::json!({ + "ai": {}, + "codingCli": { "enabledProviders": [], "mcpServer": true, "providers": {} }, + "editor": { "externalEditor": "auto" }, + "extensions": { "disabled": [] }, + "freshAgent": { "defaultPlugins": [], "enabled": false, "providers": {} }, + "logging": { "debug": false }, + "network": { "configured": true, "host": "127.0.0.1" }, + "panes": { "defaultNewPane": "ask" }, + "safety": { "autoKillIdleMinutes": 15 }, + "sidebar": { + "autoGenerateTitles": true, + "excludeFirstChatMustStart": false, + "excludeFirstChatSubstrings": [] + }, + "terminal": { "scrollback": 10000 } + }) +} + +/// Spawn a real server with a paced-page budget of [`PAGE_BUDGET`] and a +/// scrollback ring of `ring_chars` UTF-16 units (per-server registry knobs — +/// no env races between parallel tests). +async fn spawn_server(ring_chars: i64) -> String { + let auth_token = Arc::new(AUTH_TOKEN.to_string()); + let broadcast_tx = Arc::new(tokio::sync::broadcast::channel::(16).0); + let settings = + Arc::new(serde_json::from_value(test_settings_value()).expect("valid settings fixture")); + + let registry = freshell_terminal::TerminalRegistry::new(); + registry.set_paced_page_max_bytes(PAGE_BUDGET); + registry.set_scrollback_max_bytes(ring_chars); + + let state = WsState { + pane_ledger: std::sync::Arc::new(freshell_ws::pane_ledger::PaneLedger::disabled()), + layout: Default::default(), + identity: freshell_ws::identity::TerminalIdentityRegistry::new(), + terminal_meta: Default::default(), + auth_token: Arc::clone(&auth_token), + server_instance_id: Arc::new("srv-test".to_string()), + boot_id: Arc::new("boot-test".to_string()), + settings, + handshake_settings: Arc::new(tokio::sync::RwLock::new( + serde_json::from_value(test_settings_value()).expect("valid settings fixture"), + )), + broadcast_tx: Arc::clone(&broadcast_tx), + auto_resume_tx: tokio::sync::mpsc::unbounded_channel().0, + auto_resume_cancels: Default::default(), + fresh_codex: freshell_freshagent::FreshCodexState::new( + Arc::clone(&auth_token), + Arc::clone(&broadcast_tx), + serde_json::json!({ "freshAgent": { "enabled": false } }), + ), + fresh_claude: freshell_freshagent::FreshClaudeState::new(Arc::clone(&broadcast_tx)), + fresh_opencode: freshell_freshagent::FreshOpencodeState::new( + freshell_freshagent::FreshAgentState::new( + Arc::clone(&auth_token), + Arc::clone(&broadcast_tx), + ), + ), + registry, + tabs: freshell_ws::tabs::TabsRegistry::new(), + screenshots: freshell_ws::screenshot::ScreenshotBroker::new(Arc::clone(&broadcast_tx)), + subagent_interest: Default::default(), + host_stats: Default::default(), + terminals_revision: Arc::new(std::sync::atomic::AtomicI64::new(0)), + sessions_revision: Arc::new(std::sync::atomic::AtomicI64::new(0)), + cli_commands: Arc::new(Vec::new()), + shutdown: Arc::new(tokio::sync::Notify::new()), + ping_interval_ms: 30_000, + hello_timeout_ms: 5_000, + allowed_origins: Arc::new(freshell_ws::origin::default_allowed_origins()), + ws_max_payload_bytes: 64 * 1024 * 1024, + term09: freshell_ws::backpressure::Term09Config::default(), + create_protect: freshell_ws::create_limit::CreateProtectConfig::default(), + spawn_gate: std::sync::Arc::new(freshell_ws::spawn_gate::SpawnGate::new(4, 64)), + shutdown_started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + create_dedupe: std::sync::Arc::new(freshell_ws::create_dedupe::CreateDedupe::default()), + config_fallback: None, + opencode_locator: None, + codex_locator: None, + activity: None, + session_existence: std::sync::Arc::new(freshell_ws::existence::NoIndexProbe::default()), + reconcile_deferral_budget_ms: freshell_ws::reconcile::RECONCILE_DEFERRAL_BUDGET_MS_DEFAULT, + fresh_agent_respawn_counts: Default::default(), + ownership: None, + }; + + let router = freshell_ws::router(state); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("local addr"); + tokio::spawn(async move { + let _ = axum::serve(listener, router).await; + }); + + format!("ws://{addr}/ws", addr = addr) +} + +type WsClient = + tokio_tungstenite::WebSocketStream>; + +async fn connect(url: &str) -> WsClient { + let (ws, _resp) = tokio_tungstenite::connect_async(url) + .await + .expect("ws connect"); + ws +} + +/// Complete the hello handshake, optionally negotiating the paced capability +/// (and echo-reading the 4 handshake frames). +async fn hello(ws: &mut WsClient, paced: bool) { + let capabilities = if paced { + serde_json::json!({ "pacedTerminalReplayV1": true }) + } else { + serde_json::json!({}) + }; + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "hello", + "token": AUTH_TOKEN, + "protocolVersion": freshell_protocol::WS_PROTOCOL_VERSION, + "capabilities": capabilities, + }) + .to_string(), + )) + .await + .expect("send hello"); + for _ in 0..4u8 { + let msg = tokio::time::timeout(Duration::from_secs(5), ws.next()) + .await + .expect("handshake frame within timeout") + .expect("stream not ended") + .expect("no ws error"); + assert!(matches!(msg, WsMessage::Text(_))); + } +} + +async fn next_json(ws: &mut WsClient) -> serde_json::Value { + let msg = tokio::time::timeout(Duration::from_secs(10), ws.next()) + .await + .expect("frame within timeout") + .expect("stream not ended") + .expect("no ws error"); + let WsMessage::Text(text) = msg else { + panic!("expected a text frame, got {msg:?}"); + }; + serde_json::from_str(&text).expect("frame is JSON") +} + +/// Read the next JSON frame, or `None` if nothing arrives within `window`. +async fn next_json_or_timeout(ws: &mut WsClient, window: Duration) -> Option { + match tokio::time::timeout(window, ws.next()).await { + Ok(Some(Ok(WsMessage::Text(text)))) => { + Some(serde_json::from_str(&text).expect("frame is JSON")) + } + Ok(Some(Ok(WsMessage::Ping(_) | WsMessage::Pong(_)))) => None, + _ => None, + } +} + +async fn create_shell_terminal(ws: &mut WsClient, request_id: &str) -> String { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.create", + "requestId": request_id, + "mode": "shell", + "shell": "system", + }) + .to_string(), + )) + .await + .expect("send terminal.create"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while tokio::time::Instant::now() < deadline { + let value = next_json(ws).await; + if value.get("type").and_then(|v| v.as_str()) == Some("terminal.created") + && value.get("requestId").and_then(|v| v.as_str()) == Some(request_id) + { + return value + .get("terminalId") + .and_then(|v| v.as_str()) + .expect("terminal.created carries terminalId") + .to_string(); + } + } + panic!("terminal.created never arrived"); +} + +/// Send `terminal.attach` (viewport hydrate, sinceSeq 0 by default). +async fn attach(ws: &mut WsClient, terminal_id: &str, attach_request_id: &str) { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.attach", + "terminalId": terminal_id, + "intent": "viewport_hydrate", + "cols": 80, + "rows": 24, + "attachRequestId": attach_request_id, + "sinceSeq": 0, + }) + .to_string(), + )) + .await + .expect("send terminal.attach"); +} + +/// Send one continuation credit. +async fn credit(ws: &mut WsClient, terminal_id: &str, arid: &str, consumed_seq: i64) { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.replay.credit", + "terminalId": terminal_id, + "streamId": "ignored-by-server", + "attachRequestId": arid, + "consumedSeq": consumed_seq, + }) + .to_string(), + )) + .await + .expect("send terminal.replay.credit"); +} + +/// A flood whose completion is detectable by a marker that only the EXECUTED +/// printf emits (octal escapes keep the literal command text from echoing +/// the marker early — the same discipline as `term09_output_queue.rs`). +fn flood_command(lines: usize, marker: &str) -> String { + assert_eq!(marker, "FLOOD-DONE-MARKER"); + format!( + "yes 'STREAMDATA-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' | head -n {lines}; printf '\\106\\114\\117\\117\\104\\055\\104\\117\\116\\105\\055\\115\\101\\122\\113\\105\\122\\012'\n" + ) +} + +async fn send_input(ws: &mut WsClient, terminal_id: &str, data: &str) { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.input", + "terminalId": terminal_id, + "data": data, + }) + .to_string(), + )) + .await + .expect("send terminal.input"); +} + +/// Drain frames until `marker` appears in the accumulated output data. +/// Returns `(data, frames)` — the concatenated output data and every +/// terminal.output frame's seqStart (ascending as received). +async fn drain_until_marker( + ws: &mut WsClient, + marker: &str, + deadline: tokio::time::Instant, +) -> (String, Vec) { + let mut acc = String::new(); + let mut seqs = Vec::new(); + while tokio::time::Instant::now() < deadline { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + match tokio::time::timeout(remaining.max(Duration::from_millis(1)), ws.next()).await { + Ok(Some(Ok(WsMessage::Text(text)))) => { + let Ok(value) = serde_json::from_str::(&text) else { + continue; + }; + if value.get("type").and_then(|v| v.as_str()) == Some("terminal.output") { + if let Some(data) = value.get("data").and_then(|v| v.as_str()) { + acc.push_str(data); + } + seqs.push(value.get("seqStart").and_then(|v| v.as_i64()).unwrap_or(-1)); + } + if acc.contains(marker) { + return (acc, seqs); + } + } + _ => break, + } + } + (acc, seqs) +} + +/// Drive the terminal from a NON-NEGOTIATED driver connection until +/// `marker` is observed on it (the deterministic "the flood finished" +/// signal — the ring holds the full flood before the paced client attaches). +async fn flood_until_complete(url: &str, driver: &mut WsClient, terminal_id: &str, lines: usize) { + let mut observer = connect(url).await; + hello(&mut observer, false).await; + attach(&mut observer, terminal_id, "attach-observer").await; + let marker = "FLOOD-DONE-MARKER"; + send_input(driver, terminal_id, &flood_command(lines, marker)).await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let (acc, _) = drain_until_marker(&mut observer, marker, deadline).await; + assert!( + acc.contains(marker), + "the flood must complete on the observer" + ); + // Detach the observer so it stops receiving (and the terminal's + // subscriber set is clean for the paced client). + observer + .send(WsMessage::Text( + serde_json::json!({ "type": "terminal.detach", "terminalId": terminal_id }).to_string(), + )) + .await + .expect("observer detaches"); +} + +/// One attach's spontaneous delivery: the ready frame plus every output +/// frame the server sends WITHOUT any credit (the negotiated first page / +/// the non-negotiated full inline replay). Reading ends on the first QUIET +/// gap after the ready — the burst is one back-to-back admission, so a quiet +/// window means the server has nothing more to send unprompted. +async fn paced_attach_first_page( + ws: &mut WsClient, + terminal_id: &str, + arid: &str, +) -> (serde_json::Value, Vec) { + attach(ws, terminal_id, arid).await; + let mut ready = None; + let mut outputs = Vec::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while tokio::time::Instant::now() < deadline { + match next_json_or_timeout(ws, Duration::from_millis(400)).await { + None => { + if ready.is_some() { + break; // the unprompted burst is complete + } + continue; + } + Some(value) => match value.get("type").and_then(|v| v.as_str()) { + Some("terminal.attach.ready") + if value.get("attachRequestId").and_then(|v| v.as_str()) == Some(arid) => + { + ready = Some(value); + } + Some("terminal.output") => outputs.push(value), + _ => {} + }, + } + } + let ready = ready.expect("attach.ready never arrived"); + (ready, outputs) +} + +/// Negotiated attach with real scrollback: ready carries the retention +/// bounds, the first page is a BOUNDED prefix, and NO further replay frames +/// arrive while the client withholds credit. A raw-socket credit then +/// produces the next page; an out-of-window credit produces nothing. +#[tokio::test] +async fn negotiated_attach_gets_first_page_only_and_credit_gates_the_rest() { + let ring = 512 * 1024; + let url = spawn_server(ring).await; + let mut driver = connect(&url).await; + hello(&mut driver, false).await; + let terminal_id = create_shell_terminal(&mut driver, "create-paced").await; + // ~64KB of scrollback: many 4KB pages, far under the 512KB ring. + flood_until_complete(&url, &mut driver, &terminal_id, 700).await; + + let mut paced = connect(&url).await; + hello(&mut paced, true).await; + let (ready, page1) = paced_attach_first_page(&mut paced, &terminal_id, "attach-paced").await; + + // Ready bounds: the negotiated restore contract fields. + assert!( + ready["headSeq"].as_i64().unwrap_or(0) >= 1, + "honest head: {ready}" + ); + let oldest = ready["oldestRetainedSeq"] + .as_i64() + .expect("negotiated ready carries oldestRetainedSeq"); + let head = ready["headSeq"].as_i64().expect("headSeq"); + assert!( + oldest >= 1 && oldest <= head + 1, + "honest retention bound: {ready}" + ); + assert!( + ready.get("replayResetReason").is_none(), + "no retention loss in this fixture: {ready}" + ); + // The paced window description. + assert_eq!( + ready["replayToSeq"].as_i64(), + Some(head), + "replayToSeq is the fixed target: {ready}" + ); + assert_eq!( + ready["replayFromSeq"].as_i64(), + Some(1), + "the window starts at the baseline+1: {ready}" + ); + + // The first page is a bounded prefix of the window. + assert!(!page1.is_empty(), "there is replay to page"); + let last_seq = page1 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .expect("page frames carry seqEnd"); + assert!( + last_seq < head, + "the first page must not cover the whole window (head {head})" + ); + let page_bytes: usize = page1.iter().map(|f| f.to_string().len()).sum(); + assert!( + page_bytes as i64 <= PAGE_BUDGET, + "the first page honors the serialized budget: {page_bytes}" + ); + for frame in &page1 { + assert_eq!( + frame["source"], "replay", + "pages are stamped source:'replay'" + ); + assert_eq!(frame["attachRequestId"], "attach-paced"); + assert!(frame["seqStart"].as_i64().unwrap_or(0) >= 1); + } + + // WITHHOLD: no credit => no further pages (a window with no frames). + let withheld = next_json_or_timeout(&mut paced, Duration::from_millis(1500)).await; + assert!( + withheld.is_none() + || withheld + .as_ref() + .unwrap() + .get("type") + .and_then(|v| v.as_str()) + != Some("terminal.output"), + "no further replay frames may arrive without credit, got {withheld:?}" + ); + + // Beyond-window credit: ignored, produces nothing, and does not consume + // the grant (the next VALID credit still works). + credit(&mut paced, &terminal_id, "attach-paced", last_seq + 100_000).await; + let bad = next_json_or_timeout(&mut paced, Duration::from_millis(1200)).await; + assert!( + bad.is_none() + || bad.as_ref().unwrap().get("type").and_then(|v| v.as_str()) + != Some("terminal.output"), + "a beyond-window credit must produce nothing, got {bad:?}" + ); + + // Valid credit => the next page arrives. + credit(&mut paced, &terminal_id, "attach-paced", last_seq).await; + let next = next_json(&mut paced).await; + assert_eq!( + next["type"], "terminal.output", + "the credit produces the next page: {next}" + ); + let next_end = next["seqEnd"].as_i64().expect("seqEnd"); + assert!( + next_end > last_seq, + "pages ascend: page1 ends {last_seq}, next starts at {}", + next["seqStart"] + ); + assert_eq!(next["attachRequestId"], "attach-paced"); + assert_eq!(next["source"], "replay"); +} + +/// Live output produced DURING the paced replay is delivered strictly after +/// the pages covering it, in seq order, with no loss or duplication — the +/// hard "no overtaking" invariant, end to end. +#[tokio::test] +async fn live_output_during_paced_replay_never_overtakes_the_pages() { + let ring = 512 * 1024; + let url = spawn_server(ring).await; + let mut driver = connect(&url).await; + hello(&mut driver, false).await; + let terminal_id = create_shell_terminal(&mut driver, "create-live-race").await; + flood_until_complete(&url, &mut driver, &terminal_id, 400).await; + + let marker1 = "FLOOD-DONE-MARKER"; + let mut paced = connect(&url).await; + hello(&mut paced, true).await; + let (ready, page1) = paced_attach_first_page(&mut paced, &terminal_id, "attach-live").await; + let head = ready["headSeq"].as_i64().expect("headSeq"); + let first_page_last = page1 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .unwrap_or(0); + let last_seq = page1 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .unwrap_or(0); + assert!( + last_seq < head, + "the session starts mid-replay (head {head}, first page ends {last_seq}, frames {})", + page1.len() + ); + + // MORE live output while the replay is in flight (deferred — staged). + // A non-negotiated observer attaches (inline replay + direct live) and + // drives the flood to completion — the deterministic "it finished" signal. + let mut midflood = connect(&url).await; + hello(&mut midflood, false).await; + attach(&mut midflood, &terminal_id, "attach-midflood").await; + let marker2 = "FLOOD-DONE-MARKER"; + send_input(&mut midflood, &terminal_id, &flood_command(300, marker2)).await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let (mid_acc, _) = drain_until_marker(&mut midflood, marker2, deadline).await; + assert!( + mid_acc.contains(marker2), + "the mid flood completes on its own observer" + ); + drop(midflood); + + // Credit through to completion; the tail delivers the staged range + // spontaneously once the cursor reaches the target. + let mut received: Vec<(i64, String)> = page1 + .iter() + .map(|f| { + ( + f["seqStart"].as_i64().unwrap_or(0), + f["data"].as_str().unwrap_or("").to_string(), + ) + }) + .collect(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let mut credited = last_seq; + // Un-pause the session: the first page is the one outstanding page, so + // credit it before waiting for the next. + credit(&mut paced, &terminal_id, "attach-live", first_page_last).await; + while tokio::time::Instant::now() < deadline { + let Some(value) = next_json_or_timeout(&mut paced, Duration::from_secs(5)).await else { + break; + }; + if value.get("type").and_then(|v| v.as_str()) == Some("terminal.output") { + received.push(( + value["seqStart"].as_i64().unwrap_or(0), + value["data"].as_str().unwrap_or("").to_string(), + )); + let end = value["seqEnd"].as_i64().unwrap_or(0); + if end > credited { + credited = end; + credit(&mut paced, &terminal_id, "attach-live", end).await; + } + let acc: String = received.iter().map(|(_, d)| d.as_str()).collect(); + if acc.contains(marker2) { + break; + } + } + } + let acc: String = received.iter().map(|(_, d)| d.as_str()).collect(); + assert!( + acc.contains(marker1) && acc.contains(marker2), + "the session delivers both floods" + ); + + // The hard invariant: strictly ascending seqs, no duplicates, and every + // delivered frame lies at or after the FIRST page's range (nothing the + // pages cover is re-delivered, and no live frame jumped the pages). + let seqs: Vec = received.iter().map(|(s, _)| *s).collect(); + let mut sorted = seqs.clone(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(seqs.len(), sorted.len(), "no duplicate seqStarts"); + assert_eq!(seqs, sorted, "delivery is strictly in seq order"); + assert!(seqs[0] >= 1, "the first page starts at the baseline+1"); + // Every seq from the attach baseline to the end is covered exactly once. + for expected in seqs[0]..=*seqs.last().unwrap() { + assert!( + seqs.contains(&expected), + "no seq holes: {expected} missing from {:?}", + &seqs[..seqs.len().min(20)] + ); + } +} + +/// Retention expiry MID-REPLAY: the negotiated `terminal.output.gap` with +/// reason `replay_window_exceeded`, the exact lost interval, the task-2 +/// bounds fields, and continuation from the new baseline through to the +/// session's target. +#[tokio::test] +async fn mid_replay_retention_expiry_emits_the_exact_negotiated_gap_and_continues() { + // Small ring so a withheld session loses its middle to eviction. + let ring = 12 * 1024; + let url = spawn_server(ring).await; + let mut driver = connect(&url).await; + hello(&mut driver, false).await; + let terminal_id = create_shell_terminal(&mut driver, "create-expiry").await; + flood_until_complete(&url, &mut driver, &terminal_id, 100).await; + + let mut paced = connect(&url).await; + hello(&mut paced, true).await; + let (ready, page1) = paced_attach_first_page(&mut paced, &terminal_id, "attach-expiry").await; + let head = ready["headSeq"].as_i64().expect("headSeq"); + let last_seq = page1 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .expect("the first page covers something"); + assert!(last_seq < head); + + // Evict the middle of the replay window while the client withholds. + // The evictor attaches (non-negotiated — inline replay + live) to + // observe its own flood's completion deterministically. + let mut evictor = connect(&url).await; + hello(&mut evictor, false).await; + attach(&mut evictor, &terminal_id, "attach-evictor").await; + let marker2 = "FLOOD-DONE-MARKER"; + send_input(&mut evictor, &terminal_id, &flood_command(400, marker2)).await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let (acc, _) = drain_until_marker(&mut evictor, marker2, deadline).await; + assert!(acc.contains(marker2), "the evicting flood completes"); + drop(evictor); + + // Credit: the next needed frames were evicted => the negotiated gap. + credit(&mut paced, &terminal_id, "attach-expiry", last_seq).await; + let gap = loop { + let value = next_json(&mut paced).await; + match value.get("type").and_then(|v| v.as_str()) { + Some("terminal.output.gap") => break value, + Some("terminal.output") => continue, // tail leakage of pre-eviction pages + other => panic!("expected the retention gap, got {other:?} ({value})"), + } + }; + assert_eq!( + gap["reason"], "replay_window_exceeded", + "the gap names retention loss: {gap}" + ); + assert_eq!( + gap["fromSeq"].as_i64(), + Some(last_seq + 1), + "the lost interval starts at the credited cursor+1: {gap}" + ); + let gap_oldest = gap["oldestRetainedSeq"] + .as_i64() + .expect("negotiated gap carries oldestRetainedSeq"); + let gap_head = gap["headSeq"] + .as_i64() + .expect("negotiated gap carries headSeq"); + assert_eq!( + gap["toSeq"].as_i64(), + Some(gap_oldest - 1), + "the lost interval ends just before the new ring front: {gap}" + ); + assert!(gap_head >= head, "the bounds are current: {gap}"); + assert_eq!(gap["attachRequestId"], "attach-expiry"); + + // Continuation from the new baseline: the next page starts at the ring + // front and the session still reaches its original target + the live + // tail (both markers). + let mut received: Vec = Vec::new(); + let mut data = String::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let mut credited = gap["toSeq"].as_i64().unwrap(); + while tokio::time::Instant::now() < deadline { + let value = next_json_or_timeout(&mut paced, Duration::from_secs(5)).await; + let Some(value) = value else { break }; + if value.get("type").and_then(|v| v.as_str()) == Some("terminal.output") { + let seq = value["seqStart"].as_i64().unwrap_or(0); + assert!( + seq >= gap_oldest, + "continuation starts at the new baseline (ring front {gap_oldest}), got seq {seq}" + ); + received.push(seq); + data.push_str(value["data"].as_str().unwrap_or("")); + let end = value["seqEnd"].as_i64().unwrap_or(0); + if end > credited { + credited = end; + credit(&mut paced, &terminal_id, "attach-expiry", end).await; + } + if data.contains(marker2) { + break; + } + } + } + assert!( + data.contains(marker2), + "the session continues through the retained range to the live tail" + ); + let mut sorted = received.clone(); + sorted.sort_unstable(); + assert_eq!(received, sorted, "continuation pages ascend"); +} + +/// A re-attach supersedes the old session: the old generation's credit is +/// ignored (stale), the new generation's credit drives its own pages. +#[tokio::test] +async fn reattach_supersedes_the_old_paced_session() { + let ring = 512 * 1024; + let url = spawn_server(ring).await; + let mut driver = connect(&url).await; + hello(&mut driver, false).await; + let terminal_id = create_shell_terminal(&mut driver, "create-supersede").await; + flood_until_complete(&url, &mut driver, &terminal_id, 500).await; + + let mut paced = connect(&url).await; + hello(&mut paced, true).await; + let (ready1, page1) = paced_attach_first_page(&mut paced, &terminal_id, "attach-gen-1").await; + let last1 = page1 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .expect("gen-1 first page"); + assert!(last1 < ready1["headSeq"].as_i64().unwrap()); + + // Re-attach (generation 2): a fresh ready + fresh first page; the + // attach.ready supersede also discards gen-1's un-leased frames. + let (ready2, page2) = paced_attach_first_page(&mut paced, &terminal_id, "attach-gen-2").await; + assert_eq!(ready2["attachRequestId"], "attach-gen-2"); + let last2 = page2 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .expect("gen-2 first page"); + assert!( + page2.iter().all(|f| f["attachRequestId"] == "attach-gen-2"), + "gen-2's pages are stamped with gen-2's id" + ); + + // The OLD generation's credit is stale: nothing arrives. + credit(&mut paced, &terminal_id, "attach-gen-1", last1).await; + let stale = next_json_or_timeout(&mut paced, Duration::from_millis(1200)).await; + assert!( + stale.is_none() + || stale.as_ref().unwrap().get("type").and_then(|v| v.as_str()) + != Some("terminal.output"), + "a stale-generation credit must be ignored, got {stale:?}" + ); + + // The NEW generation's credit drives its own session. + credit(&mut paced, &terminal_id, "attach-gen-2", last2).await; + let next = next_json(&mut paced).await; + assert_eq!(next["type"], "terminal.output"); + assert_eq!(next["attachRequestId"], "attach-gen-2"); + assert!( + next["seqEnd"].as_i64().unwrap_or(0) > last2, + "gen-2's pages continue from its own cursor" + ); +} + +/// A credit on a NON-NEGOTIATED connection is inert: no pacing machinery +/// engages, inline delivery continues to work exactly as before. +#[tokio::test] +async fn credit_on_a_non_negotiated_connection_is_inert() { + let ring = 512 * 1024; + let url = spawn_server(ring).await; + let mut driver = connect(&url).await; + hello(&mut driver, false).await; + let terminal_id = create_shell_terminal(&mut driver, "create-plain-credit").await; + flood_until_complete(&url, &mut driver, &terminal_id, 100).await; + + let mut plain = connect(&url).await; + hello(&mut plain, false).await; + let (ready, outputs) = paced_attach_first_page(&mut plain, &terminal_id, "attach-plain").await; + // Non-negotiated: the FULL inline replay arrived immediately. + let head = ready["headSeq"].as_i64().unwrap(); + let max_seq = outputs + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .unwrap_or(0); + assert_eq!( + max_seq, head, + "a non-negotiated attach gets the whole replay inline" + ); + assert!( + ready.get("oldestRetainedSeq").is_none(), + "no contract fields for a plain connection" + ); + + // The inert credit: live output keeps flowing inline afterwards. + credit(&mut plain, &terminal_id, "attach-plain", max_seq).await; + let marker = "FLOOD-DONE-MARKER"; + send_input(&mut driver, &terminal_id, &flood_command(30, marker)).await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + let (acc, _) = drain_until_marker(&mut plain, marker, deadline).await; + assert!( + acc.contains(marker), + "live output flows normally after an inert credit" + ); +} + +/// A disconnect mid-replay leaves the terminal RUNNING and clean: a +/// subsequent (non-negotiated) attach gets the full replay and live output — +/// no leak, no wedge. +#[tokio::test] +async fn disconnect_mid_replay_leaves_the_terminal_running_and_reattachable() { + let ring = 512 * 1024; + let url = spawn_server(ring).await; + let mut driver = connect(&url).await; + hello(&mut driver, false).await; + let terminal_id = create_shell_terminal(&mut driver, "create-disconnect").await; + flood_until_complete(&url, &mut driver, &terminal_id, 400).await; + + let mut paced = connect(&url).await; + hello(&mut paced, true).await; + let (ready, page1) = paced_attach_first_page(&mut paced, &terminal_id, "attach-doomed").await; + let last_seq = page1 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .unwrap_or(0); + assert!( + last_seq < ready["headSeq"].as_i64().unwrap(), + "mid-replay before the drop" + ); + drop(paced); // the socket dies mid-replay (no detach, no credit) + + tokio::time::sleep(Duration::from_millis(300)).await; + + // A fresh non-negotiated connection attaches and gets the full replay. + let mut fresh = connect(&url).await; + hello(&mut fresh, false).await; + let (ready2, outputs) = paced_attach_first_page(&mut fresh, &terminal_id, "attach-after").await; + let head = ready2["headSeq"].as_i64().unwrap(); + let max_seq = outputs + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .unwrap_or(0); + assert_eq!( + max_seq, head, + "the full replay is intact after the dropped paced session" + ); + + // And live output still flows. + let marker = "FLOOD-DONE-MARKER"; + send_input(&mut driver, &terminal_id, &flood_command(30, marker)).await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + let (acc, _) = drain_until_marker(&mut fresh, marker, deadline).await; + assert!( + acc.contains(marker), + "live output flows after the re-attach" + ); +} diff --git a/port/contract/ws-message-inventory.json b/port/contract/ws-message-inventory.json index bfffb67af..765757ff3 100644 --- a/port/contract/ws-message-inventory.json +++ b/port/contract/ws-message-inventory.json @@ -1,6 +1,6 @@ { "clientToServer": { - "count": 41, + "count": 42, "types": [ "amplifier.activity.list", "claude.activity.list", @@ -40,6 +40,7 @@ "terminal.input", "terminal.interest", "terminal.kill", + "terminal.replay.credit", "terminal.resize", "ui.layout.sync", "ui.screenshot.result" diff --git a/port/contract/ws-protocol.schema.json b/port/contract/ws-protocol.schema.json index 2d63d3423..712e68ab0 100644 --- a/port/contract/ws-protocol.schema.json +++ b/port/contract/ws-protocol.schema.json @@ -2,7 +2,7 @@ "description": "Auto-generated from shared/ws-protocol.ts. DO NOT EDIT BY HAND. Regenerate with `npm run contract:generate`. Each entry in `schemas` is a self-contained JSON Schema for one exported Zod schema. The wire contract is frozen for the Rust port — changing it is out of scope.", "generator": "port/contract/generate-ws-contract.ts", "jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema", - "schemaCount": 83, + "schemaCount": 84, "schemas": { "AmplifierActivityListResponseSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -1343,6 +1343,43 @@ ], "type": "object" }, + { + "additionalProperties": false, + "properties": { + "attachRequestId": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "consumedSeq": { + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + }, + "streamId": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "terminalId": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "type": { + "const": "terminal.replay.credit", + "type": "string" + } + }, + "required": [ + "type", + "terminalId", + "streamId", + "attachRequestId", + "consumedSeq" + ], + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -8700,6 +8737,44 @@ ], "type": "object" }, + "TerminalReplayCreditSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "attachRequestId": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "consumedSeq": { + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + }, + "streamId": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "terminalId": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "type": { + "const": "terminal.replay.credit", + "type": "string" + } + }, + "required": [ + "type", + "terminalId", + "streamId", + "attachRequestId", + "consumedSeq" + ], + "type": "object" + }, "TerminalResizeSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, diff --git a/shared/ws-protocol.ts b/shared/ws-protocol.ts index 92f7e5dda..e77a0d4c7 100644 --- a/shared/ws-protocol.ts +++ b/shared/ws-protocol.ts @@ -1087,6 +1087,23 @@ export const TerminalInterestSchema = z.object({ }) export type TerminalInterestMessage = z.infer +/** + * Paced replay continuation credit (responsive-terminal-restore Workstream 1): + * sent by a client whose hello negotiated `pacedTerminalReplayV1` after it + * fully consumed an ordered replay page. `consumedSeq` is the last sequence + * consumed in order; `attachRequestId` scopes the credit to one attach + * generation. Additive optional — older servers accept-and-strip it and + * protocol version stays 10. + */ +export const TerminalReplayCreditSchema = z.object({ + type: z.literal('terminal.replay.credit'), + terminalId: z.string().min(1).max(512), + streamId: z.string().min(1).max(512), + attachRequestId: z.string().min(1).max(512), + consumedSeq: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), +}) +export type TerminalReplayCreditMessage = z.infer + // ── Client message discriminated union ── export const ClientMessageSchema = z.discriminatedUnion('type', [ @@ -1107,6 +1124,7 @@ export const ClientMessageSchema = z.discriminatedUnion('type', [ TerminalInputSchema, TerminalResizeSchema, TerminalKillSchema, + TerminalReplayCreditSchema, CodexActivityListSchema, OpencodeActivityListSchema, ClaudeActivityListSchema, From 0b2e7335677521dbc71829d28f7af9357d2c7534 Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sun, 20 Sep 2026 02:41:43 -0700 Subject: [PATCH 10/70] fix(ws): cancel paced sessions on legacy re-attach; tighten page budget and event pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fix commit for the paced replay core (task 003 findings): - I1: every SUCCESSFUL re-attach now cancels the connection's previous paced session for the terminal (the Legacy reply arm included) — a stale-generation credit after an arid-less legacy re-attach can no longer produce phantom pages. A failed attach (Error) still cancels nothing: the previous session keeps matching its live subscriber and deferral. Pinned end-to-end on real sockets. - M1: batch-mode page accounting now charges the once-per-batch envelope delta (the .batch type suffix + serializedBytes + the segments[] wrapper: 41 fixed + digits(budget)) on every batch-mode frame, so a page the walk believes fits the budget really fits — every produced batch contains >=1 frame, so multi-batch pages are covered too. Boundary-pinned with barrier-frame fixtures tuned to the budget edge. - M2: the ws.restore.* observability events are pinned on the real dispatch (paced_start with max_replay_bytes/page_bytes, paced_complete, all four credit verdicts) via the process-global capture rig, with a content-free assertion (no terminal data in any event field). - N1: the geometry-authorized attach path's maxReplayBytes recording is pinned inside the simultaneous-geometry test. - N3: the paced_replay harness's quiet-gap heuristic matches text frames only — a keepalive ping inside a read window no longer truncates a page read. --- crates/freshell-terminal/src/registry.rs | 193 ++++++++- crates/freshell-ws/src/terminal.rs | 21 +- crates/freshell-ws/tests/paced_replay.rs | 529 ++++++++++++++++++++++- 3 files changed, 722 insertions(+), 21 deletions(-) diff --git a/crates/freshell-terminal/src/registry.rs b/crates/freshell-terminal/src/registry.rs index 8be51108d..60ee67f7a 100644 --- a/crates/freshell-terminal/src/registry.rs +++ b/crates/freshell-terminal/src/registry.rs @@ -4151,17 +4151,32 @@ fn page_last_seq(msg: &ServerMessage) -> Option { /// stays under this, so a walk that stops at the budget never undercounts. const SEGMENT_WIRE_OVERHEAD_ESTIMATE: i64 = 96; +/// Fixed part of the once-per-batch envelope delta over the legacy +/// scaffold for the `terminal.output.batch` wire projection: the `.batch` +/// type suffix (7), the `,"serializedBytes":` key (20), and the +/// `,"segments":[` + `]` array wrapper (14). The variable part — the +/// digits of `serializedBytes` itself — is bounded by +/// [`crate::batch::digit_count`] of the page budget for any page that fits +/// it, so the walk charges `41 + digits(budget)` per batch-mode frame. +/// Charged on EVERY batch-mode frame because every produced batch contains +/// at least one frame (a multi-batch page — e.g. barrier-separated frames — +/// pays one envelope PER batch): merged frames were already over-charged +/// their full standalone envelope, so this only narrows their slack. +const BATCH_ENVELOPE_FIXED_BYTES: i64 = 41; + /// Select and project ONE bounded, ascending page of the /// `(from_seq, to_seq_inclusive]` window for `conn_id`'s subscriber. The /// caller holds the terminal lock; frames are selected BEFORE cloning (no /// full-ring snapshot per page). Packing accounts every frame at its /// STANDALONE envelope cost (the batch builder's own accounting, reusing /// `measure_serialized_json_bytes` via the incremental scaffold), plus the -/// batch-segment overhead on batch-capable subscribers — an overestimate -/// of the merged wire cost, so every produced page's real serialized bytes -/// stay within the budget. A single frame whose own envelope exceeds the -/// budget forms its own atomic single-frame page (guaranteed progress; the -/// oversize result is explicit, never silently coalesced). +/// batch-segment overhead and the once-per-batch envelope delta +/// (`serializedBytes` + the `segments[]` wrapper + the `.batch` type +/// suffix) on batch-capable subscribers — an overestimate of the merged +/// wire cost, so every produced page's real serialized bytes stay within +/// the budget. A single frame whose own envelope exceeds the budget forms +/// its own atomic single-frame page (guaranteed progress; the oversize +/// result is explicit, never silently coalesced). fn paced_page_build( s: &TerminalShared, conn_id: u64, @@ -4177,6 +4192,18 @@ fn paced_page_build( OutputSource::Replay => "replay", OutputSource::Live => "live", }; + // The batch-mode per-frame charge adds the once-per-batch envelope + // delta: a page that fits the budget serializes every payload at or + // under it, so its `serializedBytes` digits never exceed + // `digit_count(budget)` (an over-budget page is the explicit atomic + // oversize result, which is allowed to exceed). + let batch_mode_charge = if batch_mode { + SEGMENT_WIRE_OVERHEAD_ESTIMATE + + BATCH_ENVELOPE_FIXED_BYTES + + crate::batch::digit_count(budget.max(0)) as i64 + } else { + 0 + }; // First ring index with seq_start > from_seq (the ring is seq-ascending; // binary search avoids an O(ring) scan per page). @@ -4212,14 +4239,7 @@ fn paced_page_build( let escaped = crate::batch::json_escaped_len(&f.output.data) as i64; let digits = (crate::batch::digit_count(f.output.seq_start) + crate::batch::digit_count(f.output.seq_end)) as i64; - let cost = scaffold - + digits - + escaped - + if batch_mode { - SEGMENT_WIRE_OVERHEAD_ESTIMATE - } else { - 0 - }; + let cost = scaffold + digits + escaped + batch_mode_charge; if selected.is_empty() { // Always include the first frame: an over-budget frame forms // its own atomic single-frame page. @@ -5417,6 +5437,123 @@ mod tests { ); } + /// Page-budget honesty at the boundary: the batch-mode charge must + /// cover the one-frame batch ENVELOPE delta — the `.batch` type suffix, + /// the `serializedBytes` field, and the `segments[]` wrapper, which the + /// legacy-scaffold + segment estimate alone never accounted. Two + /// BARRIER frames (BEL ⇒ `turn_complete`) each form their own + /// single-frame batch, so a page carrying both pays TWO batch + /// envelopes; the budget is tuned so the pre-delta accounting admits + /// both frames while their real wire bytes exceed it. With the delta + /// charged, the walk emits each frame as its own single-frame page — + /// every page the walk believes fits the budget really fits. + #[test] + fn paced_batch_page_accounting_covers_the_envelope_at_the_budget_boundary() { + let reg = TerminalRegistry::new(); + reg.insert_headless("T", "S"); + let data_a = format!("{}{}", "a".repeat(120), '\u{0007}'); + let data_b = format!("{}{}", "b".repeat(120), '\u{0007}'); + reg.feed("T", frame(1, &data_a, "S")); + reg.feed("T", frame(2, &data_b, "S")); + + // Size the budget from the crate's own envelope accounting: the + // pre-delta per-frame charge (scaffold + seq digits + escaped data + + // the segment estimate) for BOTH frames plus a 5-byte margin — the + // exact boundary corner where an un-accounted envelope delta can + // push a page's real serialized bytes over the budget. + let scaffold = crate::batch::legacy_envelope_scaffold_bytes( + "T", + "S", + Some("batch-paced"), + Some("replay"), + ) as i64; + let accounted_pre_delta = |data: &str| { + scaffold + + crate::batch::digit_count(1) as i64 + + crate::batch::digit_count(2) as i64 + + crate::batch::json_escaped_len(data) as i64 + + SEGMENT_WIRE_OVERHEAD_ESTIMATE + }; + let budget = accounted_pre_delta(&data_a) + accounted_pre_delta(&data_b) + 5; + reg.set_paced_page_max_bytes(budget); + + let (sink, _seen) = collector(); + let out = reg.attach( + "T", + 2, + sink, + Some("batch-paced".into()), + 0, + true, + true, + None, + None, + None, + ); + let start = out.paced.expect("paced session"); + + // The first page honors the budget with REAL serialized bytes. + assert!( + start.session.page_bytes as i64 <= budget, + "the first page's real serialized bytes ({}) must stay within the \ + budget ({budget}) — the batch envelope delta must be accounted", + start.session.page_bytes + ); + assert_eq!( + page_serialized_bytes(&start.first_page) as i64, + start.session.page_bytes as i64, + "the session's page_bytes is the page's real serialized size" + ); + // The envelope corner is real for this fixture: one single-frame + // batch page's wire cost exceeds the pre-delta accounted charge. + assert!( + page_serialized_bytes(&start.first_page) as i64 > accounted_pre_delta(&data_a), + "the fixture must actually exercise the envelope delta corner" + ); + // The walk stopped before frame 2: the first page is a single + // frame's own single-frame batch page. + let page1 = page_seq_data(&start.first_page); + assert_eq!( + page1.iter().map(|(s, _)| *s).collect::>(), + vec![1], + "the delta-inclusive charge leaves frame 2 for its own page" + ); + assert!( + start.first_page.len() == 1, + "one single-frame batch payload" + ); + assert!(matches!( + start.first_page[0], + ServerMessage::TerminalOutputBatch(_) + )); + + // Frame 2 arrives as its own bounded single-frame page. + match reg.next_replay_page("T", 2, start.session.page_end, start.session.target, budget) { + PacedPage::Frames { + messages, + end_seq, + serialized_bytes, + } => { + assert_eq!(end_seq, 2); + assert_eq!(messages.len(), 1, "one single-frame batch payload"); + assert!( + serialized_bytes as i64 <= budget, + "every page the walk emits within its accounting stays \ + within the budget: {serialized_bytes} > {budget}" + ); + assert!( + serialized_bytes as i64 > accounted_pre_delta(&data_b), + "frame 2's real page cost also exceeds the pre-delta charge" + ); + } + other => panic!("expected frame 2's page, got {other:?}"), + } + match reg.next_replay_page("T", 2, 2, start.session.target, budget) { + PacedPage::Done => {} + other => panic!("a drained window reads Done, got {other:?}"), + } + } + /// Pages ascend within the serialized budget to the FIXED attach-time /// target — output produced after the attach never extends the replay /// range (it is tail-phase delivery instead). @@ -6925,7 +7062,10 @@ mod tests { false, None, None, - None, + // TERM-07 seam: the geometry-authorized path threads the + // client's replay-budget request just like the plain + // path (pinned by the subscriber-recording assert below). + Some(256 * 1024), TerminalAttachIntent::ViewportHydrate, 131, 48, @@ -6978,6 +7118,31 @@ mod tests { matches!(geometry, (131, 48, 1) | (67, 30, 1)), "the final dimensions must belong to the one successful first claim: {geometry:?}" ); + // TERM-07 seam on the GEOMETRY-AUTHORIZED path (attach_with_geometry + // threads max_replay_bytes into the same attach_to_shared as the + // plain path): each subscriber records its own attach's value — + // a's request verbatim, b's absence as None — with no + // delivery-behavior change. + let (recorded_a, recorded_b) = { + let inner = reg.inner.lock().unwrap(); + let handle = inner.terminals.get("T").unwrap(); + let s = handle.shared.lock().unwrap(); + ( + s.subscribers + .get(&1) + .expect("conn 1 subscriber") + .max_replay_bytes, + s.subscribers + .get(&2) + .expect("conn 2 subscriber") + .max_replay_bytes, + ) + }; + assert_eq!(recorded_a, Some(256 * 1024)); + assert_eq!( + recorded_b, None, + "an attach without maxReplayBytes records absence" + ); } #[test] diff --git a/crates/freshell-ws/src/terminal.rs b/crates/freshell-ws/src/terminal.rs index 6d0e9f8ef..6462191fe 100644 --- a/crates/freshell-ws/src/terminal.rs +++ b/crates/freshell-ws/src/terminal.rs @@ -1489,6 +1489,21 @@ async fn handle_client_text( // Observability inputs captured before the attach moves. let requested_since_seq = attach.since_seq.unwrap_or(0); let attach_max_replay_bytes = attach.max_replay_bytes; + // Responsive-terminal-restore binding point 6 (supersede): + // EVERY successful re-attach for the same (connection, + // terminal) cancels the connection's previous paced session + // for that terminal — a cancelled session's late credits are + // ignored as a stale generation. This must cover the LEGACY + // reply too (an arid-less re-attach from a negotiated + // connection, or an attach to an exited terminal): the + // registry already replaced the subscriber, so a surviving + // ws-layer session would be fed by a NEW subscriber's ring + // reads and could still produce phantom pages for a + // stale-generation credit. A FAILED attach (Error) cancels + // nothing — the previous session still matches its live + // subscriber and deferral. Each arm cancels BEFORE the paced + // insert below, so the new session is never the one removed. + let attach_terminal_id = attach.terminal_id.clone(); let attached = match handle_attach( attach, state, @@ -1498,8 +1513,12 @@ async fn handle_client_text( paced_terminal_replay_v1, ) { AttachReply::Error(err) => send(ws_tx, &err).await, - AttachReply::Legacy => true, + AttachReply::Legacy => { + paced_sessions.cancel(&attach_terminal_id); + true + } AttachReply::Paced(start) => { + paced_sessions.cancel(&attach_terminal_id); // The paced replay core (responsive-terminal-restore // W1): sink the first page (the registry produced it // under the attach lock), emit the session start, and diff --git a/crates/freshell-ws/tests/paced_replay.rs b/crates/freshell-ws/tests/paced_replay.rs index a597d198f..130070cf3 100644 --- a/crates/freshell-ws/tests/paced_replay.rs +++ b/crates/freshell-ws/tests/paced_replay.rs @@ -19,6 +19,7 @@ //! baseline; a re-attach supersedes the old session; a disconnect //! mid-replay leaves the terminal running and re-attachable. +use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; @@ -28,6 +29,162 @@ use tokio_tungstenite::tungstenite::Message as WsMessage; use freshell_ws::WsState; +// ── capturing tracing layer (dev-only test facility). PROCESS-GLOBAL by +// deliberate choice (the diag01_lifecycle_events.rs `global_capture` / +// invariants.rs e08g pattern, extended with `record_u64` so the paced +// events' u64 fields — `page_bytes`, `pages` — are captured too): a +// thread-local `set_default` capture is UNSOUND for callsites shared with +// sibling tests running in parallel — tracing-core caches each callsite's +// Interest process-wide on first registration, and a subscriber-less +// sibling thread executing a shared emission site first (e.g. +// `credit_on_a_non_negotiated_connection_is_inert` firing the +// `non_negotiated` callsite) caches `Interest::never`, so a thread-local +// capture then never sees its OWN thread's emissions (kata 59nb). One +// global subscriber sees every thread's events; every read below MUST +// filter by the per-test-unique `terminal_id` because ALL tests in this +// binary share the vec. ───────────────────────────────────────────────── + +use std::sync::Mutex; +use tracing::field::{Field, Visit}; +use tracing::{Event, Subscriber}; +use tracing_subscriber::layer::{Context, SubscriberExt}; +use tracing_subscriber::Layer; + +#[derive(Debug, Clone, Default)] +struct CapturedEvent { + message: String, + fields: BTreeMap, +} + +#[derive(Default)] +struct FieldVisitor { + message: String, + fields: BTreeMap, +} + +impl Visit for FieldVisitor { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + let rendered = format!("{value:?}"); + if field.name() == "message" { + self.message = rendered; + } else { + self.fields.insert(field.name().to_string(), rendered); + } + } + + fn record_str(&mut self, field: &Field, value: &str) { + if field.name() == "message" { + self.message = value.to_string(); + } else { + self.fields + .insert(field.name().to_string(), value.to_string()); + } + } + + fn record_i64(&mut self, field: &Field, value: i64) { + self.fields + .insert(field.name().to_string(), value.to_string()); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.fields + .insert(field.name().to_string(), value.to_string()); + } +} + +struct CaptureLayer { + events: Arc>>, +} + +impl Layer for CaptureLayer { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + let mut visitor = FieldVisitor::default(); + event.record(&mut visitor); + self.events + .lock() + .expect("capture lock") + .push(CapturedEvent { + message: visitor.message, + fields: visitor.fields, + }); + } +} + +/// Process-global capture for this test binary (first caller installs; +/// `get_or_init` is the synchronization). This binary installs no other +/// global subscriber; `.expect()` turns any future second installer into an +/// immediate diagnosable panic instead of a silently-empty capture. +fn global_capture() -> Arc>> { + static EVENTS: std::sync::OnceLock>>> = std::sync::OnceLock::new(); + Arc::clone(EVENTS.get_or_init(|| { + let events = Arc::new(Mutex::new(Vec::new())); + let layer = CaptureLayer { + events: Arc::clone(&events), + }; + let subscriber = tracing_subscriber::registry().with(layer); + tracing::subscriber::set_global_default(subscriber) + .expect("this test binary installs exactly one global subscriber"); + events + })) +} + +/// Poll the capture until an event for THIS test's terminal (the vec is +/// shared by every test in the binary — the unique `terminal_id` is the +/// per-test discriminator) with this message AND `fields[field] == value` +/// lands (or the 5s deadline passes). The `ws.restore.credit` events share +/// one message name, so the verdict `status` field is the selector. +async fn wait_for_restore_event( + events: &Arc>>, + terminal_id: &str, + message: &str, + field: &str, + value: &str, +) -> Option { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + { + let captured = events.lock().unwrap(); + if let Some(found) = captured.iter().find(|e| { + e.message == message + && e.fields.get("terminal_id").map(String::as_str) == Some(terminal_id) + && e.fields.get(field).map(String::as_str) == Some(value) + }) { + return Some(found.clone()); + } + } + if tokio::time::Instant::now() >= deadline { + return None; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +/// [`wait_for_restore_event`] without the extra field selector — for the +/// one-event-per-terminal messages (`ws.restore.paced_start`, +/// `ws.restore.paced_complete`). +async fn wait_for_restore_event_of_terminal( + events: &Arc>>, + terminal_id: &str, + message: &str, +) -> Option { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + loop { + { + let captured = events.lock().unwrap(); + if let Some(found) = captured.iter().find(|e| { + e.message == message + && e.fields.get("terminal_id").map(String::as_str) == Some(terminal_id) + }) { + return Some(found.clone()); + } + } + if tokio::time::Instant::now() >= deadline { + return None; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + const AUTH_TOKEN: &str = "s3cr3t-token-abcdef"; /// Deliberately small page budget: deterministic multi-page fixtures with /// small floods (the production default is 128 KiB). @@ -184,14 +341,28 @@ async fn next_json(ws: &mut WsClient) -> serde_json::Value { serde_json::from_str(&text).expect("frame is JSON") } -/// Read the next JSON frame, or `None` if nothing arrives within `window`. +/// Read the next JSON frame, or `None` if no frame at all arrives within +/// `window`. A keepalive Ping/Pong landing INSIDE the window is NOT +/// silence — the burst-complete heuristic treats only text frames as +/// signal, so a server ping mid-read cannot truncate a page read (the +/// 30s ping interval makes this rare vs the ~1-2s bursts, but the harness +/// must not depend on that timing). async fn next_json_or_timeout(ws: &mut WsClient, window: Duration) -> Option { - match tokio::time::timeout(window, ws.next()).await { - Ok(Some(Ok(WsMessage::Text(text)))) => { - Some(serde_json::from_str(&text).expect("frame is JSON")) + let deadline = tokio::time::Instant::now() + window; + loop { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + return None; + } + match tokio::time::timeout(remaining, ws.next()).await { + Ok(Some(Ok(WsMessage::Text(text)))) => { + return Some(serde_json::from_str(&text).expect("frame is JSON")); + } + // Control frames never count as the burst's end: keep waiting + // for text until the window actually elapses. + Ok(Some(Ok(WsMessage::Ping(_) | WsMessage::Pong(_)))) => continue, + _ => return None, } - Ok(Some(Ok(WsMessage::Ping(_) | WsMessage::Pong(_)))) => None, - _ => None, } } @@ -241,6 +412,25 @@ async fn attach(ws: &mut WsClient, terminal_id: &str, attach_request_id: &str) { .expect("send terminal.attach"); } +/// Send `terminal.attach` WITHOUT an `attachRequestId` — the uncorrelated +/// legacy shape: even a negotiated connection falls back to the inline +/// full-replay path (credits cannot be correlated without a generation key). +async fn attach_without_arid(ws: &mut WsClient, terminal_id: &str) { + ws.send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.attach", + "terminalId": terminal_id, + "intent": "viewport_hydrate", + "cols": 80, + "rows": 24, + "sinceSeq": 0, + }) + .to_string(), + )) + .await + .expect("send arid-less terminal.attach"); +} + /// Send one continuation credit. async fn credit(ws: &mut WsClient, terminal_id: &str, arid: &str, consumed_seq: i64) { ws.send(WsMessage::Text( @@ -375,6 +565,40 @@ async fn paced_attach_first_page( (ready, outputs) } +/// The LEGACY (non-paced) attach burst: the ready frame (matched by +/// terminalId — this attach carries no `attachRequestId` to match) plus +/// every output frame the server sends spontaneously (the full inline +/// replay), read to the first quiet gap after the ready. +async fn legacy_attach_inline_replay( + ws: &mut WsClient, + terminal_id: &str, +) -> (serde_json::Value, Vec) { + let mut ready = None; + let mut outputs = Vec::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while tokio::time::Instant::now() < deadline { + match next_json_or_timeout(ws, Duration::from_millis(400)).await { + None => { + if ready.is_some() { + break; // the inline burst is complete + } + continue; + } + Some(value) => match value.get("type").and_then(|v| v.as_str()) { + Some("terminal.attach.ready") + if value.get("terminalId").and_then(|v| v.as_str()) == Some(terminal_id) => + { + ready = Some(value); + } + Some("terminal.output") | Some("terminal.output.batch") => outputs.push(value), + _ => {} + }, + } + } + let ready = ready.expect("attach.ready never arrived"); + (ready, outputs) +} + /// Negotiated attach with real scrollback: ready carries the retention /// bounds, the first page is a BOUNDED prefix, and NO further replay frames /// arrive while the client withholds credit. A raw-socket credit then @@ -763,6 +987,85 @@ async fn reattach_supersedes_the_old_paced_session() { ); } +/// The binding supersede rule covers EVERY successful re-attach, not just +/// the paced one: an arid-less LEGACY re-attach (the negotiated fallback +/// shape) must also cancel the connection's previous paced session for the +/// terminal — a credit for the superseded generation must produce NOTHING +/// (no `terminal.output`, no `terminal.output.batch`), and the connection +/// must keep working afterwards. +#[tokio::test] +async fn legacy_reattach_cancels_the_stale_paced_session() { + let ring = 512 * 1024; + let url = spawn_server(ring).await; + let mut driver = connect(&url).await; + hello(&mut driver, false).await; + let terminal_id = create_shell_terminal(&mut driver, "create-legacy-supersede").await; + flood_until_complete(&url, &mut driver, &terminal_id, 400).await; + + // Generation 1: the paced attach starts a session whose first page is + // a bounded prefix (the session is ACTIVE, mid-replay). + let mut paced = connect(&url).await; + hello(&mut paced, true).await; + let (ready1, page1) = paced_attach_first_page(&mut paced, &terminal_id, "attach-gen-1").await; + let last1 = page1 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .expect("gen-1 first page"); + assert!( + last1 < ready1["headSeq"].as_i64().unwrap(), + "the session is mid-replay (supersede-able)" + ); + + // The arid-less LEGACY re-attach: the whole window replays inline. + attach_without_arid(&mut paced, &terminal_id).await; + let (ready2, outputs) = legacy_attach_inline_replay(&mut paced, &terminal_id).await; + assert!( + ready2 + .get("attachRequestId") + .and_then(|v| v.as_str()) + .is_none(), + "the re-attach carried no attachRequestId: {ready2}" + ); + let head = ready2["headSeq"].as_i64().unwrap(); + let max_seq = outputs + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .unwrap_or(0); + assert_eq!( + max_seq, head, + "the legacy re-attach replays the whole window inline" + ); + + // The superseded generation's credit (its arid + in-window consumedSeq): + // it must be ignored — no output frame of ANY kind may be produced. + credit(&mut paced, &terminal_id, "attach-gen-1", last1).await; + let stale = next_json_or_timeout(&mut paced, Duration::from_millis(1500)).await; + let is_output_frame = |v: &serde_json::Value| { + matches!( + v.get("type").and_then(|t| t.as_str()), + Some("terminal.output") | Some("terminal.output.batch") + ) + }; + assert!( + stale.as_ref().map(|v| !is_output_frame(v)).unwrap_or(true), + "a credit for the session superseded by the legacy re-attach must \ + produce nothing, got {stale:?}" + ); + + // The connection is not wedged by the cancel: live output keeps + // flowing to the re-attached (legacy) subscriber. + let marker = "FLOOD-DONE-MARKER"; + send_input(&mut driver, &terminal_id, &flood_command(30, marker)).await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + let (acc, _) = drain_until_marker(&mut paced, marker, deadline).await; + assert!( + acc.contains(marker), + "live output flows normally after the superseded credit" + ); +} + /// A credit on a NON-NEGOTIATED connection is inert: no pacing machinery /// engages, inline delivery continues to work exactly as before. #[tokio::test] @@ -858,3 +1161,217 @@ async fn disconnect_mid_replay_leaves_the_terminal_running_and_reattachable() { "live output flows after the re-attach" ); } + +/// The `ws.restore.*` observability contract, pinned end-to-end on the real +/// dispatch: `ws.restore.paced_start` carries its identifiers/measurements +/// (including `maxReplayBytes`), `ws.restore.paced_complete` closes the +/// session, and all FOUR `ws.restore.credit` verdicts (accepted / +/// stale_generation / beyond_window / non_negotiated) are emitted with their +/// status field — and NO event ever carries terminal CONTENT (identifiers +/// and measurements only). +#[tokio::test] +async fn restore_observability_events_are_emitted_content_free() { + let events = global_capture(); + let ring = 512 * 1024; + let url = spawn_server(ring).await; + let mut driver = connect(&url).await; + hello(&mut driver, false).await; + let terminal_id = create_shell_terminal(&mut driver, "create-observability").await; + flood_until_complete(&url, &mut driver, &terminal_id, 100).await; + + // The negotiated attach carries a maxReplayBytes request (the TERM-07 + // seam) — it must ride the paced_start event. + let mut paced = connect(&url).await; + hello(&mut paced, true).await; + paced + .send(WsMessage::Text( + serde_json::json!({ + "type": "terminal.attach", + "terminalId": terminal_id, + "intent": "viewport_hydrate", + "cols": 80, + "rows": 24, + "attachRequestId": "attach-ev", + "sinceSeq": 0, + "maxReplayBytes": 262144, + }) + .to_string(), + )) + .await + .expect("send attach with maxReplayBytes"); + let (ready, page1) = paced_attach_first_page(&mut paced, &terminal_id, "attach-ev").await; + let head = ready["headSeq"].as_i64().expect("headSeq"); + let last_seq = page1 + .iter() + .map(|f| f["seqEnd"].as_i64().unwrap_or(0)) + .max() + .expect("first page"); + let page_bytes: u64 = page1.iter().map(|f| f.to_string().len() as u64).sum(); + + // paced_start: identifiers + measurements, maxReplayBytes included. + let start_ev = + wait_for_restore_event_of_terminal(&events, &terminal_id, "ws.restore.paced_start") + .await + .expect("ws.restore.paced_start is emitted on the negotiated attach"); + assert_eq!( + start_ev.fields.get("attach_request_id").map(String::as_str), + Some("attach-ev") + ); + assert_eq!( + start_ev.fields.get("requested_since").map(String::as_str), + Some("0") + ); + assert_eq!( + start_ev.fields.get("effective_since").map(String::as_str), + Some("0") + ); + assert_eq!( + start_ev.fields.get("target").map(String::as_str), + Some(head.to_string().as_str()), + "target is the attach-time head" + ); + assert_eq!( + start_ev.fields.get("max_replay_bytes").map(String::as_str), + Some("Some(262144)"), + "the TERM-07 seam value rides the event (Debug of Option)" + ); + assert_eq!( + start_ev.fields.get("page_bytes").map(String::as_str), + Some(page_bytes.to_string().as_str()), + "page_bytes is the first page's real serialized size" + ); + + // beyond_window: a consumedSeq past the last-sent page's end is ignored. + credit(&mut paced, &terminal_id, "attach-ev", last_seq + 100_000).await; + let beyond = wait_for_restore_event( + &events, + &terminal_id, + "ws.restore.credit", + "status", + "beyond_window", + ) + .await + .expect("the beyond-window credit is observed"); + assert_eq!( + beyond.fields.get("terminal_id").map(String::as_str), + Some(terminal_id.as_str()) + ); + assert_eq!( + beyond + .fields + .get("consumed_seq") + .and_then(|v| v.parse::().ok()), + Some(last_seq + 100_000) + ); + + // stale_generation: a credit for an attachRequestId no active session + // holds is a stale generation. + credit(&mut paced, &terminal_id, "attach-bogus", last_seq).await; + let stale = wait_for_restore_event( + &events, + &terminal_id, + "ws.restore.credit", + "status", + "stale_generation", + ) + .await + .expect("the stale-generation credit is observed"); + assert_eq!( + stale.fields.get("terminal_id").map(String::as_str), + Some(terminal_id.as_str()) + ); + + // accepted: a valid credit produces the next page... + credit(&mut paced, &terminal_id, "attach-ev", last_seq).await; + let accepted = wait_for_restore_event( + &events, + &terminal_id, + "ws.restore.credit", + "status", + "accepted", + ) + .await + .expect("the valid credit is observed as accepted"); + assert_eq!( + accepted + .fields + .get("consumed_seq") + .and_then(|v| v.parse::().ok()), + Some(last_seq) + ); + // ...and the page actually arrives (the event describes real behavior). + let next = next_json(&mut paced).await; + assert_eq!(next["type"], "terminal.output"); + + // Drive the session to completion; the tail drains un-credited and the + // registry's atomic clear completes the session. + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let mut credited = next["seqEnd"].as_i64().unwrap_or(last_seq); + credit(&mut paced, &terminal_id, "attach-ev", credited).await; + while tokio::time::Instant::now() < deadline { + let Some(value) = next_json_or_timeout(&mut paced, Duration::from_secs(5)).await else { + break; + }; + if value.get("type").and_then(|v| v.as_str()) == Some("terminal.output") { + let end = value["seqEnd"].as_i64().unwrap_or(0); + if end > credited { + credited = end; + credit(&mut paced, &terminal_id, "attach-ev", end).await; + } + } + } + let complete = + wait_for_restore_event_of_terminal(&events, &terminal_id, "ws.restore.paced_complete") + .await + .expect("ws.restore.paced_complete closes the session"); + assert_eq!( + complete.fields.get("attach_request_id").map(String::as_str), + Some("attach-ev") + ); + assert_eq!( + complete.fields.get("last_seq").map(String::as_str), + Some(credited.to_string().as_str()), + "last_seq is the session's final cursor" + ); + assert!( + complete + .fields + .get("pages") + .and_then(|v| v.parse::().ok()) + .is_some_and(|pages| pages >= 2), + "pages counts every page the session produced" + ); + + // non_negotiated: a credit from a connection that never negotiated the + // paced capability is inert and observed as such. + credit(&mut driver, &terminal_id, "attach-ev", 0).await; + let non_negotiated = wait_for_restore_event( + &events, + &terminal_id, + "ws.restore.credit", + "status", + "non_negotiated", + ) + .await + .expect("the non-negotiated connection's credit is observed as inert"); + assert_eq!( + non_negotiated.fields.get("terminal_id").map(String::as_str), + Some(terminal_id.as_str()) + ); + + // Identifiers/measurements only: NO terminal content ever leaks into a + // ws.restore.* event for this terminal (the flood payload and its + // marker must be absent from every event field). + let captured = events.lock().unwrap(); + for event in captured.iter().filter(|e| { + e.message.starts_with("ws.restore.") + && e.fields.get("terminal_id").map(String::as_str) == Some(terminal_id.as_str()) + }) { + for (name, value) in &event.fields { + assert!( + !value.contains("STREAMDATA") && !value.contains("FLOOD-DONE-MARKER"), + "terminal content leaked into ws.restore.{name}={value}" + ); + } + } +} From 10af7d2e712cc84fadb09d2e6c97ab5df98b529a Mon Sep 17 00:00:00 2001 From: Dan Shapiro <3732858+danshapiro@users.noreply.github.com> Date: Sun, 20 Sep 2026 03:38:24 -0700 Subject: [PATCH 11/70] feat(client): consume paced replay with write-queue credit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Client side of responsive-terminal-restore Workstream 1, gated on the pacedTerminalReplayV1 capability echo (no echo -> byte-identical legacy wire behavior): - Negotiated attaches (fresh and delta) send replayPageBytes and omit maxReplayBytes; all viewportHydrateReplayOptions sites pass the negotiation flag. - New src/lib/paced-replay-consumption.ts tracks the ordered consumption frontier per attach generation: it advances when a frame is applied through the write queue (completeParserAppliedFrame) or fully consumed by a null-screen-effect pre-parser (startup probes, OSC52, turn signals); unknown mutations keep the quarantine and forfeit credit. The frontier is distinct from the parser-applied checkpoint, which still refuses filtered/lost ranges. - terminal.replay.credit is sent at most once per write-queue drain tick (coalesced via the queue's onDrain plus a flush task for filtered-only advances), clamped to the highest received seq and the session-window target from attach.ready. - attach.ready contract fields (requestedSinceSeq, effectiveSinceSeq, oldestRetainedSeq, replayResetReason) are recorded as terminal.restore.paced_ready perf audit events; negotiated replay_window_exceeded gaps render an accessible aria-live notice (unknown-bounds semantics for absent fields, recorded as terminal.restore.retention_gap) and never trigger the opencode replacement kill — live output continues. --- src/components/TerminalView.tsx | 248 +++++++++- src/components/terminal-view-utils.ts | 6 + src/lib/paced-replay-consumption.ts | 135 ++++++ .../TerminalView.lifecycle.test.tsx | 452 ++++++++++++++++++ .../components/terminal-view-utils.test.ts | 20 + .../terminal/terminal-write-queue.test.ts | 94 ++++ .../lib/paced-replay-consumption.test.ts | 169 +++++++ 7 files changed, 1107 insertions(+), 17 deletions(-) create mode 100644 src/lib/paced-replay-consumption.ts create mode 100644 test/unit/client/lib/paced-replay-consumption.test.ts diff --git a/src/components/TerminalView.tsx b/src/components/TerminalView.tsx index 5ec7c8e54..7dbad93f3 100644 --- a/src/components/TerminalView.tsx +++ b/src/components/TerminalView.tsx @@ -120,6 +120,14 @@ import { type AttachSeqState, type OutputBatchAcceptedSegment, } from '@/lib/terminal-attach-seq-state' +import { + beginPacedReplayConsumption, + pacedReplayConsumeThrough, + pacedReplayMarkReceived, + pacedReplayNextCredit, + pacedReplayOnReady, + type PacedReplayConsumptionState, +} from '@/lib/paced-replay-consumption' import { useMobile } from '@/hooks/useMobile' import { usePaneFocusAdoption } from '@/hooks/usePaneFocusAdoption' import { useKeyboardInset } from '@/hooks/useKeyboardInset' @@ -218,7 +226,11 @@ const TOUCH_SCROLL_PIXELS_PER_LINE = 18 const LIGHT_THEME_MIN_CONTRAST_RATIO = 4.5 const DEFAULT_MIN_CONTRAST_RATIO = 1 const MAX_LAST_SENT_VIEWPORT_CACHE_ENTRIES = 200 -const TRUNCATED_REPLAY_BYTES = 128 * 1024 +// One replay page (responsive-terminal-restore Workstream 1): the legacy +// maxReplayBytes truncation budget and the paced replayPageBytes request are +// deliberately the same 128 KiB — the paced path pages what the legacy path +// truncated. +const REPLAY_PAGE_BYTES = 128 * 1024 const INPUT_BLOCKED_NOTICE_THROTTLE_MS = 2000 const TERMINAL_OUTPUT_BATCH_BARRIER_REASONS = new Set([ 'control', @@ -275,10 +287,18 @@ const xtermLogger: ILogger = { }, } -function viewportHydrateReplayOptions(content?: TerminalPaneContent | null): { maxReplayBytes: number } | undefined { +function viewportHydrateReplayOptions( + content?: TerminalPaneContent | null, + pacedReplay?: boolean, +): { maxReplayBytes: number } | { replayPageBytes: number } | undefined { + if (pacedReplay) { + // Negotiated: page-sized replay delivery on every hydrate, no byte-budget + // truncation (the paced path pages the whole retained window). + return { replayPageBytes: REPLAY_PAGE_BYTES } + } return content?.mode === 'opencode' ? undefined - : { maxReplayBytes: TRUNCATED_REPLAY_BYTES } + : { maxReplayBytes: REPLAY_PAGE_BYTES } } function buildSessionAssociationContentUpdates( @@ -488,6 +508,9 @@ type AttachTerminalOptions = { suppressNextMatchingResize?: boolean skipPreAttachFit?: boolean maxReplayBytes?: number + /** Negotiated paced attaches only (viewportHydrateReplayOptions): the + * page-size request carried instead of maxReplayBytes. */ + replayPageBytes?: number priority?: TerminalAttachPriority sinceSeq?: number } @@ -656,6 +679,16 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te && window.__FRESHELL_TEST_HARNESS__?.isTerminalNetworkEffectsSuppressed?.(paneId) === true const [isAttaching, setIsAttaching] = useState(false) const [truncatedHistoryGap, setTruncatedHistoryGap] = useState<{ fromSeq: number; toSeq: number } | null>(null) + // Honest incomplete-history state (responsive-terminal-restore): a + // NEGOTIATED retention gap's visible, accessible notice. `headSeq` / + // `oldestRetainedSeq` are null when the gap omitted them — absence means + // UNKNOWN BOUNDS, never non-negotiation. + const [retentionLossNotice, setRetentionLossNotice] = useState<{ + fromSeq: number + toSeq: number + headSeq: number | null + oldestRetainedSeq: number | null + } | null>(null) const [backgroundHydrationTriggered, setBackgroundHydrationTriggered] = useState(false) const wasCreatedFreshRef = useRef(paneContent.kind === 'terminal' && paneContent.status === 'creating') const [pendingLinkUri, setPendingLinkUri] = useState(null) @@ -1106,6 +1139,80 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te quarantineRepairRef.current = null }, []) + // ── Paced terminal replay consumption (responsive-terminal-restore + // Workstream 1, client side) ── + // All paced behavior is gated on the CURRENT connection's capability echo; + // absent (or a ws client without the accessor) → today's exact wire + // behavior everywhere. + const isPacedReplayNegotiated = useCallback((): boolean => { + const capabilities = typeof ws.getServerCapabilities === 'function' + ? ws.getServerCapabilities() + : undefined + return capabilities?.pacedTerminalReplayV1 === true + }, [ws]) + + // The consumption frontier for the CURRENT attach generation only — replaced + // by the next attach, absent for non-negotiated attaches. + const pacedReplayRef = useRef(null) + + const advancePacedReplayConsumption = useCallback((attachRequestId: string | undefined, seqEnd: number) => { + const paced = pacedReplayRef.current + if (!paced || !attachRequestId || paced.attachRequestId !== attachRequestId) return + const next = pacedReplayConsumeThrough(paced, seqEnd) + if (next !== paced) { + pacedReplayRef.current = next + } + }, []) + + const markPacedReplayReceived = useCallback((attachRequestId: string | undefined, seqEnd: number) => { + const paced = pacedReplayRef.current + if (!paced || !attachRequestId || paced.attachRequestId !== attachRequestId) return + const next = pacedReplayMarkReceived(paced, seqEnd) + if (next !== paced) { + pacedReplayRef.current = next + } + }, []) + + const flushPacedReplayCredit = useCallback(() => { + const paced = pacedReplayRef.current + if (!paced) return + const activeAttach = currentAttachRef.current + if (!activeAttach || activeAttach.requestId !== paced.attachRequestId) return + const streamId = activeAttach.streamId + if (typeof streamId !== 'string' || streamId.length === 0) return + const decision = pacedReplayNextCredit(paced) + if (decision.state !== paced) { + pacedReplayRef.current = decision.state + } + if (!decision.credit) return + ws.send({ + type: 'terminal.replay.credit', + terminalId: decision.credit.terminalId, + streamId, + attachRequestId: decision.credit.attachRequestId, + consumedSeq: decision.credit.consumedSeq, + }) + }, [ws]) + + // A consumption advance with no write of its own (a fully pre-filtered + // frame) rides the write queue as a task so the SAME drain-tick coalescing + // applies: the credit flushes once per drain, never per frame. Armed for + // the current paced attach generation only — non-negotiated panes keep + // today's queue traffic byte-identical. + const schedulePacedReplayCreditFlush = useCallback(( + outputSource: TerminalOutputSource, + attachRequestId: string | undefined, + ) => { + const paced = pacedReplayRef.current + if (!paced || !attachRequestId || paced.attachRequestId !== attachRequestId) return + const queue = writeQueueRef.current + if (!queue) { + flushPacedReplayCredit() + return + } + queue.enqueueTask(() => {}, { mode: outputSource, generation: attachRequestId }) + }, [flushPacedReplayCredit]) + const recordTerminalPerfAuditEvent = useCallback((event: string, data: Record = {}) => { const payload = Object.fromEntries(Object.entries({ event, @@ -1142,7 +1249,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te clearQuarantineRepair(attachRequestId) attachTerminalRef.current?.(terminalId, 'viewport_hydrate', { clearViewportFirst: true, - ...viewportHydrateReplayOptions(contentRef.current), + ...viewportHydrateReplayOptions(contentRef.current, isPacedReplayNegotiated()), }) return } @@ -1174,7 +1281,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te timedOut: false, timer: setTimeout(poll, QUARANTINE_REPAIR_POLL_MS), } - }, [clearQuarantineRepair, recordTerminalPerfAuditEvent]) + }, [clearQuarantineRepair, isPacedReplayNegotiated, recordTerminalPerfAuditEvent]) const markParserAppliedFrame = useCallback((terminalId: string | undefined, seq: number, attachContext?: { requestId: string @@ -2281,6 +2388,12 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te surfaceWritesSinceFreshRef.current = 0 const writeQueue = createTerminalWriteQueue({ terminalInstanceId, + // One paced-replay credit per drain tick (Workstream 1): the flush is + // idempotent (lastSentCreditSeq guard), so non-paced panes and drains + // with no frontier movement are no-ops. + onDrain: () => { + flushPacedReplayCredit() + }, onItemApplied: (item) => { surfaceWritesSinceFreshRef.current += 1 // Coupled clear (plan round-3): the marker-bearing attach's own @@ -3082,6 +3195,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te setIsAttaching(true) setTruncatedHistoryGap(null) + setRetentionLossNotice(null) // Startup probes must not leak across attach generations. resetStartupProbeParser() @@ -3110,6 +3224,14 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te pendingReason: opts?.priority === 'background' ? 'background_catchup' : 'initial_hydrate', } + // Paced replay consumption (Workstream 1): arm the frontier for THIS + // generation only on a negotiated connection — the next attach replaces + // it; non-negotiated attaches never arm one. + const pacedReplayNegotiated = isPacedReplayNegotiated() + pacedReplayRef.current = pacedReplayNegotiated + ? beginPacedReplayConsumption({ terminalId: tid, attachRequestId, sinceSeq }) + : null + currentAttachRef.current = { requestId: attachRequestId, intent: effectiveIntent, @@ -3175,7 +3297,13 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te // fence — terminal.attach participates in the coordinator (a queued // cross-device attach is generation-fenced server-side). ownerFence: selectPaneOwnerFence(appStore.getState(), contentRef.current ?? {}) ?? undefined, - ...(opts?.maxReplayBytes ? { maxReplayBytes: opts.maxReplayBytes } : {}), + // Paced replay negotiation (Workstream 1): negotiated attaches — + // fresh AND delta, with or without explicit options — request + // page-sized delivery and never carry the legacy truncation budget. + // Non-negotiated stays byte-identical to today. + ...(pacedReplayNegotiated + ? { replayPageBytes: opts?.replayPageBytes ?? REPLAY_PAGE_BYTES } + : opts?.maxReplayBytes ? { maxReplayBytes: opts.maxReplayBytes } : {}), ...(claimSurfaceReset ? { surfaceReset: true } : {}), })) rememberSentViewport(tid, cols, rows) @@ -3199,6 +3327,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te clearQuarantineRepair, getCheckpointDeltaReplayDecision, getTerminalCheckpointStreamId, + isPacedReplayNegotiated, recordTerminalPerfAuditEvent, resetParserAppliedSurface, scheduleQuarantineRepair, @@ -3239,13 +3368,13 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te } else { attachTerminal(tid, 'viewport_hydrate', { clearViewportFirst: true, - ...viewportHydrateReplayOptions(currentContent), + ...viewportHydrateReplayOptions(currentContent, isPacedReplayNegotiated()), }) } dispatch(consumePaneRefreshRequest({ tabId, paneId, requestId: request.requestId })) return true - }, [attachTerminal, dispatch, paneId, registerForBackgroundHydration, suppressNetworkEffects, tabId, ws]) + }, [attachTerminal, dispatch, isPacedReplayNegotiated, paneId, registerForBackgroundHydration, suppressNetworkEffects, tabId, ws]) // Apply settings changes useEffect(() => { @@ -3298,14 +3427,14 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te suppressNextMatchingResize: true, skipPreAttachFit: true, ...(revealPlan.intent === 'viewport_hydrate' - ? viewportHydrateReplayOptions(contentRef.current) + ? viewportHydrateReplayOptions(contentRef.current, isPacedReplayNegotiated()) : undefined), }) return } requestTerminalLayout({ fit: true, resize: true }) } - }, [hidden, isTerminal, paneId, requestTerminalLayout, tabId, attachTerminal, getCheckpointDeltaReplayDecision]) + }, [hidden, isTerminal, paneId, requestTerminalLayout, tabId, attachTerminal, getCheckpointDeltaReplayDecision, isPacedReplayNegotiated]) // Background hydration: triggered by the hydration queue for hidden tabs useEffect(() => { @@ -3325,9 +3454,9 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te attachTerminal(tid, 'viewport_hydrate', { clearViewportFirst: true, priority: 'background', - ...viewportHydrateReplayOptions(contentRef.current), + ...viewportHydrateReplayOptions(contentRef.current, isPacedReplayNegotiated()), }) - }, [backgroundHydrationTriggered, attachTerminal, getCheckpointDeltaReplayDecision]) + }, [backgroundHydrationTriggered, attachTerminal, getCheckpointDeltaReplayDecision, isPacedReplayNegotiated]) // Create or attach to backend terminal useEffect(() => { @@ -3880,6 +4009,12 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te const nextSeqState = markParserAppliedSeq(seqStateRef.current, input.parserAppliedSeq) applySeqState(nextSeqState) markParserAppliedFrame(tid, nextSeqState.parserAppliedSeq, activeAttach) + // Paced replay consumption (Workstream 1): the write-queue applied + // this frame — the consumption frontier advances independent of the + // parser-applied checkpoint's quarantine clamping (the checkpoint + // still refuses filtered/lost ranges; the frontier is about + // consumption, not surface state). + advancePacedReplayConsumption(input.attachRequestId, input.parserAppliedSeq) if (input.completedAttach) { completeAttachGeneration({ attachRequestId: input.attachRequestId, @@ -4004,6 +4139,16 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te || !inputBytesEqualSubmission || !submission.submittedBytesEqualInput ) { + // Paced replay consumption (Workstream 1): a frame the + // null-screen-effect pre-parsers fully consumed (nothing reached + // the queue, no replay-discard mutation) IS consumed — the + // frontier advances without any xterm write. Partial or unknown + // mutations keep today's quarantine and forfeit the range's + // credit (the server's retention/expiry handling covers it). + if (!submission.submittedWrite && inputBytesEqualSubmission) { + advancePacedReplayConsumption(input.attachRequestId, input.seqEnd) + schedulePacedReplayCreditFlush(input.outputSource, input.attachRequestId) + } applySeqState(markOutputRangeUnapplied(seqStateRef.current, { fromSeq: input.seqStart, toSeq: input.seqEnd, @@ -4210,6 +4355,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te const completedAttachOnBatch = !batchDecision.state.pendingReplay && (Boolean(previousSeqState.pendingReplay) || previousSeqState.awaitingFreshSequence) applySeqState(batchDecision.state) + markPacedReplayReceived(msg.attachRequestId, batchSeqEnd) const containsBarrier = batchSegments.some((segment) => segment.barrier) if (!containsBarrier) { @@ -4301,6 +4447,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te const completedAttachOnFrame = !frameDecision.state.pendingReplay && (Boolean(previousSeqState.pendingReplay) || previousSeqState.awaitingFreshSequence) applySeqState(frameDecision.state) + markPacedReplayReceived(msg.attachRequestId, msg.seqEnd) submitAcceptedOutput({ raw: msg.data || '', seqStart: msg.seqStart, @@ -4332,9 +4479,16 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te // Only show "load more" when the server confirms the gap is from // byte-budget truncation (recoverable), not ring overflow (data gone). + const pacedReplayNegotiated = isPacedReplayNegotiated() const isTruncatedReplay = msg.reason === 'replay_budget_exceeded' && seqStateRef.current.pendingReplay - const isUnrecoverableOpenCodeViewportHydrate = msg.reason === 'replay_window_exceeded' + // Negotiated retention gaps NEVER trigger the opencode replacement + // kill (responsive-terminal-restore): the paced server continues + // from what is retained and the pane shows the honest + // incomplete-history notice instead. The kill stays for old servers + // (no echo ⇒ no new restore semantics). + const isUnrecoverableOpenCodeViewportHydrate = !pacedReplayNegotiated + && msg.reason === 'replay_window_exceeded' && currentAttachRef.current?.intent === 'viewport_hydrate' && currentAttachRef.current.sinceSeq === 0 && contentRef.current?.mode === 'opencode' @@ -4345,6 +4499,27 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te if (isTruncatedReplay) { setTruncatedHistoryGap({ fromSeq: msg.fromSeq, toSeq: msg.toSeq }) + } else if (pacedReplayNegotiated && msg.reason === 'replay_window_exceeded') { + // Honest incomplete-history state on the paced path: some earlier + // output is gone; live output continues. Absent bounds fields + // mean UNKNOWN BOUNDS (the terminal may have vanished at gap + // emission) — never non-negotiation. + const gapHeadSeq = typeof msg.headSeq === 'number' ? msg.headSeq : null + const gapOldestRetainedSeq = typeof msg.oldestRetainedSeq === 'number' ? msg.oldestRetainedSeq : null + setRetentionLossNotice({ + fromSeq: msg.fromSeq, + toSeq: msg.toSeq, + headSeq: gapHeadSeq, + oldestRetainedSeq: gapOldestRetainedSeq, + }) + recordTerminalPerfAuditEvent('terminal.restore.retention_gap', { + terminalId: tid, + attachRequestId: msg.attachRequestId, + fromSeq: msg.fromSeq, + toSeq: msg.toSeq, + headSeq: gapHeadSeq, + oldestRetainedSeq: gapOldestRetainedSeq, + }) } else { const reason = msg.reason === 'replay_window_exceeded' ? 'reconnect window exceeded' @@ -4355,6 +4530,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te const gapDecision = onOutputGap(previousSeqState, { fromSeq: msg.fromSeq, toSeq: msg.toSeq }) const nextSeqState = gapDecision.state applySeqState(nextSeqState) + markPacedReplayReceived(msg.attachRequestId, msg.toSeq) resetParserAppliedSurface(parserAppliedSeqRef.current) if (gapDecision.requiresSurfaceQuarantine) { recordTerminalPerfAuditEvent('terminal.catchup.surface_quarantined', { @@ -4480,7 +4656,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te updateContent({ streamId: undefined }) attachTerminal(tid, 'viewport_hydrate', { clearViewportFirst: true, - ...viewportHydrateReplayOptions(contentRef.current), + ...viewportHydrateReplayOptions(contentRef.current, isPacedReplayNegotiated()), }) return } @@ -4515,7 +4691,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te resetParserAppliedSurface(parserAppliedSeqRef.current) attachTerminal(tid, 'viewport_hydrate', { clearViewportFirst: true, - ...viewportHydrateReplayOptions(contentRef.current), + ...viewportHydrateReplayOptions(contentRef.current, isPacedReplayNegotiated()), }) return } @@ -4567,6 +4743,29 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te replayToSeq: msg.replayToSeq, }) applySeqState(nextSeqState) + // Paced replay consumption (Workstream 1): record the session + // window end (replayToSeq on a paced ready describes the SESSION + // window — the fixed catch-up target — never a single page's + // bounds) and the restore-contract fields. Legacy-path ready frames + // on a negotiated connection (an already-Exited terminal) carry the + // contract fields too; their credits are inert server-side. + const pacedReadyState = pacedReplayRef.current + if (pacedReadyState && msg.attachRequestId === pacedReadyState.attachRequestId) { + pacedReplayRef.current = pacedReplayOnReady(pacedReadyState, { + replayToSeq: msg.replayToSeq, + }) + recordTerminalPerfAuditEvent('terminal.restore.paced_ready', { + terminalId: tid, + attachRequestId: msg.attachRequestId, + requestedSinceSeq: msg.requestedSinceSeq, + effectiveSinceSeq: msg.effectiveSinceSeq, + oldestRetainedSeq: msg.oldestRetainedSeq, + replayResetReason: msg.replayResetReason, + headSeq: msg.headSeq, + replayFromSeq: msg.replayFromSeq, + replayToSeq: msg.replayToSeq, + }) + } setIsAttaching(Boolean(nextSeqState.pendingReplay)) if (!nextSeqState.pendingReplay) { // Completion-clear edge for empty-tracker / no-replay attaches @@ -5628,7 +5827,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te ? 'keepalive_delta' : 'viewport_hydrate' attachTerminal(currentTerminalId, intent, intent === 'viewport_hydrate' - ? viewportHydrateReplayOptions(contentRef.current) + ? viewportHydrateReplayOptions(contentRef.current, isPacedReplayNegotiated()) : undefined) // One-shot reconcile notice (attach/corrected/duplicate verdicts): // render it on the attach that the verdict fold re-fired, then @@ -6086,8 +6285,23 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te )} + {retentionLossNotice && ( + // Honest incomplete-history state (responsive-terminal-restore): a + // negotiated retention gap's accessible notice — live output keeps + // flowing below it. +
+ Some earlier terminal output is no longer available on the server. Live output continues. +
+ )} {truncatedHistoryGap && ( -
+
)} + {recoveryExhausted && ( + // Bounded automatic recovery (responsive-terminal-restore WS2): the + // visible, accessible retry state. Automatic re-attach cycling was + // stopped after repeated attempts with no restore progress; the + // terminal content below is PRESERVED and an explicit retry resumes. +
+ Terminal restore is not making progress. What is on screen is unchanged. + +
+ )} {truncatedHistoryGap && (
void + /** + * Surface-mutation ledger (responsive-terminal-restore WS2): fired for + * EVERY completed write item — INCLUDING stale generations (the item's + * bytes were already submitted to the surface when it went in flight, so a + * stale completion is still a mutation even though onItemApplied rightly + * skips it). Never fired for tasks. Used by the quarantine repair to prove + * the surface still matches its last checkpoint. + */ + onWriteCompleted?: (item: { mode: TerminalWriteQueueMode; generation: string | undefined }) => void budgetMs?: number now?: () => number requestFrame?: (cb: FrameRequestCallback) => number @@ -140,6 +149,7 @@ export function createTerminalWriteQueue(args: TerminalWriteQueueArgs): Terminal for (const callback of item.callbacks) callback() args.onItemApplied?.({ mode: item.mode, generation: item.generation }) } + args.onWriteCompleted?.({ mode: item.mode, generation: item.generation }) } finally { scope.complete() decrementInFlightWrites(item.generation) diff --git a/src/lib/terminal-cursor.ts b/src/lib/terminal-cursor.ts index 812619272..b0198d63f 100644 --- a/src/lib/terminal-cursor.ts +++ b/src/lib/terminal-cursor.ts @@ -22,6 +22,22 @@ export type TerminalSurfaceCheckpointIdentity = { serverBootId?: string } +/** + * Surface scoping (responsive-terminal-restore WS2): checkpoints are keyed by + * the surface that rendered them — a pane/surface instance — so sibling panes + * rendering the same terminal cannot borrow or overwrite each other's + * rendered progress. The pane id is the stable per-surface discriminator (the + * attachRequestId already embeds `${paneId}:`). An absent scope addresses + * the legacy terminal-keyed store and never sees scoped entries. + */ +export type TerminalSurfaceScope = { paneId?: string } + +const SURFACE_KEY_SEPARATOR = '::' + +function surfaceStoreKey(scope: TerminalSurfaceScope | undefined, terminalId: string): string { + return scope?.paneId ? `${scope.paneId}${SURFACE_KEY_SEPARATOR}${terminalId}` : terminalId +} + type CursorMap = Record const MAX_ENTRIES = 500 @@ -88,6 +104,9 @@ function sanitizeCheckpoint( surfaceEpoch: normalizeSeq(candidate.surfaceEpoch), attachRequestId: candidate.attachRequestId, parserAppliedSeq: normalizeSeq(candidate.parserAppliedSeq), + ...(typeof candidate.surfaceCoverageSeq === 'number' + ? { surfaceCoverageSeq: normalizeSeq(candidate.surfaceCoverageSeq) } + : {}), cols: normalizeSeq(candidate.cols), rows: normalizeSeq(candidate.rows), geometryEpoch: normalizeSeq(candidate.geometryEpoch), @@ -98,25 +117,34 @@ function sanitizeCheckpoint( parserIdle: candidate.parserIdle === true, }) - if (checkpoint.parserAppliedSeq <= 0) return null + // A checkpoint needs SOME resumable position: rendered content (applied) + // or accounted coverage past a null-screen-effect filtered prefix. + if (checkpoint.parserAppliedSeq <= 0 && (checkpoint.surfaceCoverageSeq ?? 0) <= 0) return null return checkpoint } +function terminalIdFromStoreKey(key: string): string { + const separatorIndex = key.lastIndexOf(SURFACE_KEY_SEPARATOR) + return separatorIndex === -1 ? key : key.slice(separatorIndex + SURFACE_KEY_SEPARATOR.length) +} + function sanitizeMap(raw: unknown): CursorMap { if (!raw || typeof raw !== 'object') return {} const input = raw as Record const out: CursorMap = {} - for (const [terminalId, value] of Object.entries(input)) { - if (!terminalId) continue + for (const [storeKey, value] of Object.entries(input)) { + if (!storeKey) continue if (!value || typeof value !== 'object') continue const candidate = value as Record - const checkpoint = sanitizeCheckpoint(terminalId, candidate.checkpoint) + // Scoped keys embed the terminal id (`${paneId}::${terminalId}`); the + // checkpoint must still self-identify with that terminal. + const checkpoint = sanitizeCheckpoint(terminalIdFromStoreKey(storeKey), candidate.checkpoint) const updatedAt = normalizeTimestamp(candidate.updatedAt) if (!checkpoint || updatedAt <= 0) continue - out[terminalId] = { checkpoint, updatedAt } + out[storeKey] = { checkpoint, updatedAt } } return out @@ -236,24 +264,47 @@ function sameCheckpointSurface( && a.bufferType === b.bufferType } +function coveragePositionOf(checkpoint: TerminalSurfaceCheckpoint): number { + const coverage = checkpoint.surfaceCoverageSeq + // Zero coverage = no contiguous coverage beyond the applied position (a + // rendered tail with a lost prefix): fall back to applied, exactly the + // pre-cursor ranking behavior. + if (typeof coverage !== 'number' || !Number.isFinite(coverage) || coverage <= 0) { + return checkpoint.parserAppliedSeq + } + return Math.floor(coverage) +} + function chooseCheckpoint( existing: TerminalSurfaceCheckpoint | undefined, next: TerminalSurfaceCheckpoint, ): TerminalSurfaceCheckpoint { if (!existing) return next if (!sameCheckpointSurface(existing, next)) return next + // Overwrite prevention: keep the most advanced reconstruction position — + // coverage first (it subsumes the rendered baseline), applied as the + // legacy tiebreak. A regressed save can never clobber real progress. + const existingCoverage = coveragePositionOf(existing) + const nextCoverage = coveragePositionOf(next) + if (existingCoverage > nextCoverage) return existing + if (existingCoverage < nextCoverage) return next if (existing.parserAppliedSeq > next.parserAppliedSeq) return existing return next } -function saveCheckpointEntry(checkpoint: TerminalSurfaceCheckpoint): void { - if (!checkpoint.terminalId || checkpoint.parserAppliedSeq <= 0) return +function saveCheckpointEntry( + checkpoint: TerminalSurfaceCheckpoint, + scope: TerminalSurfaceScope | undefined, +): void { + if (!checkpoint.terminalId) return + if (checkpoint.parserAppliedSeq <= 0 && (checkpoint.surfaceCoverageSeq ?? 0) <= 0) return const map = ensureLoaded() const now = Date.now() - const existing = map[checkpoint.terminalId] + const storeKey = surfaceStoreKey(scope, checkpoint.terminalId) + const existing = map[storeKey] const nextCheckpoint = chooseCheckpoint(existing?.checkpoint, checkpoint) - map[checkpoint.terminalId] = { checkpoint: nextCheckpoint, updatedAt: now } + map[storeKey] = { checkpoint: nextCheckpoint, updatedAt: now } const shouldPrune = Object.keys(map).length > MAX_ENTRIES || now - lastPruneAt >= PRUNE_INTERVAL_MS @@ -271,9 +322,10 @@ function saveCheckpointEntry(checkpoint: TerminalSurfaceCheckpoint): void { export function loadTerminalSurfaceCheckpoint( terminalId: string, identity: TerminalSurfaceCheckpointIdentity, + scope?: TerminalSurfaceScope, ): TerminalSurfaceCheckpoint | null { if (!terminalId) return null - const entry = ensureLoaded()[terminalId] + const entry = ensureLoaded()[surfaceStoreKey(scope, terminalId)] if (!entry) return null const checkpoint = entry.checkpoint @@ -285,8 +337,11 @@ export function loadTerminalSurfaceCheckpoint( return { ...checkpoint } } -export function saveTerminalSurfaceCheckpoint(input: TerminalSurfaceCheckpoint): void { - saveCheckpointEntry(createTerminalSurfaceCheckpoint(input)) +export function saveTerminalSurfaceCheckpoint( + input: TerminalSurfaceCheckpoint, + scope?: TerminalSurfaceScope, +): void { + saveCheckpointEntry(createTerminalSurfaceCheckpoint(input), scope) } export function loadTerminalCursor(terminalId: string): number { @@ -302,8 +357,15 @@ export function saveTerminalCursor(terminalId: string, seq: number): void { export function clearTerminalCursor(terminalId: string): void { if (!terminalId) return const map = ensureLoaded() - if (!map[terminalId]) return - delete map[terminalId] + // Surface-scoped entries live under `${paneId}::${terminalId}` keys — a + // terminal-wide clear must remove every pane's entry for the terminal. + const matchingKeys = Object.keys(map).filter((key) => ( + map[key]?.checkpoint.terminalId === terminalId + )) + if (matchingKeys.length === 0) return + for (const key of matchingKeys) { + delete map[key] + } if (persistTimer) { clearTimeout(persistTimer) persistTimer = null diff --git a/src/lib/terminal-recovery-accounting.ts b/src/lib/terminal-recovery-accounting.ts new file mode 100644 index 000000000..1ceeb8697 --- /dev/null +++ b/src/lib/terminal-recovery-accounting.ts @@ -0,0 +1,131 @@ +/** + * Bounded automatic recovery accounting (responsive-terminal-restore + * Workstream 2): one state object per terminal pane, gating automatic + * attach/hydrate cycling. + * + * The progress signal is the SURFACE COVERAGE CURSOR — genuine consumption of + * stream content (applied or fully pre-filtered). Receiving `attach.ready` or + * another reconnect is NOT progress. The accounting resets on genuine + * coverage progress or an explicit user retry; it NEVER triggers a kill, a + * replacement, or an identity change — reaching the bound only stops + * automatic attaches and shows a visible retry state. + * + * Bounds (constants per repo conventions, overridable by tests via `now`): + * - at most TERMINAL_RECOVERY_MAX_ATTEMPTS consecutive progressless attempts; + * - a progressless streak older than TERMINAL_RECOVERY_NO_PROGRESS_DEADLINE_MS + * with at least two attempts sent (a single stray re-attach after an idle + * pane is never blocked by the deadline alone). + */ +export const TERMINAL_RECOVERY_MAX_ATTEMPTS = 3 +export const TERMINAL_RECOVERY_NO_PROGRESS_DEADLINE_MS = 30_000 + +export type TerminalRecoveryAccounting = { + /** Highest coverage position that ever reset the streak. */ + lastProgressSeq: number + /** Consecutive attach/hydrate attempts sent without coverage progress. */ + attempts: number + /** When the current progressless streak started (null when clean). */ + streakStartedAt: number | null + /** True once a bound was hit; blocks every automatic attempt until reset. */ + exhausted: boolean + /** + * False until the pane's INITIAL hydration attach is observed. The first + * attach of a pane is initial hydration, not recovery cycling — it never + * counts toward the bound (a reconcile-verdict episode legitimately fires + * several deliberate re-attaches right after it). + */ + initialAttachConsumed: boolean +} + +export function createTerminalRecoveryAccounting(): TerminalRecoveryAccounting { + return { + lastProgressSeq: 0, + attempts: 0, + streakStartedAt: null, + exhausted: false, + initialAttachConsumed: false, + } +} + +/** + * Record genuine coverage progress. Idempotent: re-reporting the current + * position changes nothing; only an ADVANCE resets the streak (and clears + * exhaustion — live output on a stuck pane proves the surface is not dead). + */ +export function recordRecoveryProgress( + state: TerminalRecoveryAccounting, + coverageSeq: number, + _now: number, +): TerminalRecoveryAccounting { + if (coverageSeq <= state.lastProgressSeq) return state + return { + ...state, + lastProgressSeq: coverageSeq, + attempts: 0, + streakStartedAt: null, + exhausted: false, + } +} + +/** + * Decide whether ONE automatic attach attempt may proceed, folding in any + * progress-since-last-attempt first. `allowed: false` means the bound was + * reached: the caller stops automatic re-attach cycling and shows the + * visible retry state (the accounting stays exhausted until progress or an + * explicit retry resets it). + */ +export function beginRecoveryAttempt( + state: TerminalRecoveryAccounting, + input: { coverageSeq: number; now: number }, +): { state: TerminalRecoveryAccounting; allowed: boolean } { + const progressed = recordRecoveryProgress(state, input.coverageSeq, input.now) + if (!state.initialAttachConsumed) { + // The pane's INITIAL hydration attach: never counts toward the bound. + return { + state: { ...progressed, initialAttachConsumed: true }, + allowed: true, + } + } + if (progressed !== state) { + return { state: progressed, allowed: true } + } + if (state.exhausted) { + return { state, allowed: false } + } + const attempts = state.attempts + const streakStartedAt = state.streakStartedAt ?? input.now + const deadlineExceeded = attempts >= 2 + && input.now - streakStartedAt >= TERMINAL_RECOVERY_NO_PROGRESS_DEADLINE_MS + if (attempts >= TERMINAL_RECOVERY_MAX_ATTEMPTS || deadlineExceeded) { + return { state: { ...state, exhausted: true }, allowed: false } + } + return { + state: { + ...state, + attempts: attempts + 1, + streakStartedAt, + }, + allowed: true, + } +} + +/** + * Explicit user retry (the visible retry control, or an explicit pane + * refresh): resets the accounting to clean and re-arms automatic recovery + * from the current coverage position. + */ +export function resetRecoveryAccounting( + state: TerminalRecoveryAccounting, + coverageSeq: number, + _now: number, +): TerminalRecoveryAccounting { + return { + lastProgressSeq: coverageSeq, + attempts: 0, + streakStartedAt: null, + exhausted: false, + // An explicit retry happens on an already-hydrated pane: the initial + // attach exemption stays consumed. + initialAttachConsumed: true, + } +} diff --git a/src/lib/terminal-surface-checkpoint.ts b/src/lib/terminal-surface-checkpoint.ts index 86b530a49..ebac9a4b3 100644 --- a/src/lib/terminal-surface-checkpoint.ts +++ b/src/lib/terminal-surface-checkpoint.ts @@ -9,6 +9,17 @@ export type TerminalSurfaceCheckpoint = { surfaceEpoch: number attachRequestId: string parserAppliedSeq: number + /** + * Surface-coverage cursor (responsive-terminal-restore WS1/WS2): + * reconstruction-safe contiguous coverage of the stream on this surface — + * applied frames AND fully-consumed null-screen-effect filtered frames + * advance it; unknown mutations, lost ranges, and locally-unapplied + * ranges pin it. Distinct from the strict `parserAppliedSeq`, which never + * advances across a filtered or lost range. Optional for legacy + * checkpoints persisted before the field existed (they resume from their + * applied position). + */ + surfaceCoverageSeq?: number cols: number rows: number geometryEpoch: number @@ -57,12 +68,38 @@ function normalizeNonNegativeInteger(value: number): number { return Math.max(0, Math.floor(value)) } +/** + * Normalized coverage position of a checkpoint. Zero coverage is "no + * contiguous coverage information beyond the applied position" (e.g. a + * rendered tail with a lost prefix): the resume position falls back to the + * strict applied position, exactly the pre-cursor behavior. A POSITIVE + * coverage is the reconstruction-safe resume position (it runs ahead of + * applied past null-screen-effect filtered ranges). + */ +function coverageSeqOf(checkpoint: TerminalSurfaceCheckpoint): number { + const coverage = checkpoint.surfaceCoverageSeq + if (typeof coverage !== 'number' || !Number.isFinite(coverage) || coverage <= 0) { + return checkpoint.parserAppliedSeq + } + return Math.floor(coverage) +} + function normalizeCheckpoint(input: TerminalSurfaceCheckpoint): TerminalSurfaceCheckpoint { + const parserAppliedSeq = normalizeNonNegativeInteger(input.parserAppliedSeq) + const surfaceCoverageSeq = input.surfaceCoverageSeq return { ...input, streamId: input.streamId ?? null, surfaceEpoch: normalizeNonNegativeInteger(input.surfaceEpoch), - parserAppliedSeq: normalizeNonNegativeInteger(input.parserAppliedSeq), + parserAppliedSeq, + // Coverage is normalized against the applied position: absent → the + // applied position (legacy resume behavior); present → clamped to ≥ 0. + // It is intentionally NOT clamped down to parserAppliedSeq — the coverage + // cursor legitimately runs AHEAD of applied past null-screen-effect + // filtered ranges. + surfaceCoverageSeq: typeof surfaceCoverageSeq === 'number' && Number.isFinite(surfaceCoverageSeq) + ? Math.max(0, Math.floor(surfaceCoverageSeq)) + : parserAppliedSeq, cols: normalizeNonNegativeInteger(input.cols), rows: normalizeNonNegativeInteger(input.rows), geometryEpoch: normalizeNonNegativeInteger(input.geometryEpoch), @@ -140,9 +177,18 @@ export function canUseCheckpointForDeltaReplay( if (current.requireParserIdle && !saved.parserIdle) { return { ok: false, reason: 'parser_busy' } } - if (saved.parserAppliedSeq <= 0) { + // Eligibility is fulfilled by the COVERAGE cursor (`no_applied_sequence`'s + // role: never resume a surface from nothing). The strict applied check's + // integrity role is preserved by the both-zero rejection below: applied>0 + // alone remains a legacy fallback position, and coverage>0 with applied=0 + // is a legitimately covered surface (a filtered prefix renders nothing but + // is fully accounted). + if (saved.parserAppliedSeq <= 0 && coverageSeqOf(saved) <= 0) { return { ok: false, reason: 'no_applied_sequence' } } - return { ok: true, sinceSeq: saved.parserAppliedSeq } + // Delta resumes request their since position from the COVERAGE cursor — + // never from the strict applied position a filter pinned below + // already-rendered content. + return { ok: true, sinceSeq: coverageSeqOf(saved) } } diff --git a/test/unit/client/components/TerminalView.lifecycle.test.tsx b/test/unit/client/components/TerminalView.lifecycle.test.tsx index 88e37f6f0..5428c8983 100644 --- a/test/unit/client/components/TerminalView.lifecycle.test.tsx +++ b/test/unit/client/components/TerminalView.lifecycle.test.tsx @@ -6038,11 +6038,11 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-from-ready', serverInstanceId: 'server-attach-stream', - })?.parserAppliedSeq).toBe(1) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(1) expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: null, serverInstanceId: 'server-attach-stream', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() }) it('accepts live output after a terminal.stream.changed control message without trusting the old stream', async () => { @@ -6082,7 +6082,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-before-change', serverInstanceId: 'server-active-stream-change', - })?.parserAppliedSeq).toBe(1) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(1) act(() => { messageHandler!({ @@ -6114,11 +6114,11 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-before-change', serverInstanceId: 'server-active-stream-change', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-after-change', serverInstanceId: 'server-active-stream-change', - })?.parserAppliedSeq).toBe(2) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(2) }) it('treats mismatched replay after a stream change as a completing lost range', async () => { @@ -6205,7 +6205,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-after-change', serverInstanceId: 'server-stale-replay-stream-change', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() }) it('rejects a warm-delta attach when attach-ready reports a different stream id', async () => { @@ -6247,7 +6247,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-before-rotation', serverInstanceId: 'server-stream-rotation', - })?.parserAppliedSeq).toBe(1) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(1) wsMocks.send.mockClear() act(() => { @@ -6290,7 +6290,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-after-rotation', serverInstanceId: 'server-stream-rotation', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.attach', terminalId, @@ -6361,7 +6361,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-geometry', serverInstanceId: 'server-geometry-authority', - })?.parserAppliedSeq).toBe(1) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(1) wsMocks.send.mockClear() act(() => { @@ -6479,7 +6479,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-active', serverInstanceId: 'server-stream-mismatch', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() wsMocks.send.mockClear() act(() => { @@ -6722,7 +6722,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-hole', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() wsMocks.send.mockClear() act(() => { @@ -6793,7 +6793,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-malformed-numbers', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() act(() => { messageHandler!({ @@ -6810,7 +6810,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-malformed-numbers', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() wsMocks.send.mockClear() act(() => { @@ -6857,7 +6857,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-surrogate-split', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() }) it('rejects terminal.output.batch when segment data disagrees with offsets', async () => { @@ -6935,7 +6935,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-invalid-fail-closed', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() expect(bridge.snapshot().perfEvents).toContainEqual(expect.objectContaining({ event: 'terminal.catchup.surface_quarantined', terminalId, @@ -6985,7 +6985,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-invalid-overlap-tail', - })?.parserAppliedSeq).toBe(10) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(10) term.write.mockClear() act(() => { @@ -7020,7 +7020,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-invalid-overlap-tail', - })?.parserAppliedSeq).toBe(10) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(10) wsMocks.send.mockClear() act(() => { @@ -7080,7 +7080,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-barrier-coalesced', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() act(() => { rafCallbacks.shift()?.(16) @@ -7090,7 +7090,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-barrier-coalesced', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() act(() => { delayedCallbacks[0]?.callback() @@ -7099,7 +7099,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-barrier-coalesced', - })?.parserAppliedSeq).toBe(3) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(3) }) it('does not checkpoint across a stripped middle batch segment when adjacent renderable segments coalesce', async () => { @@ -7150,7 +7150,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-stripped-middle-coalesced', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() act(() => { rafCallbacks.shift()?.(16) @@ -7160,7 +7160,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-stripped-middle-coalesced', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() act(() => { delayedCallbacks[0]?.callback() @@ -7169,7 +7169,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-stripped-middle-coalesced', - })?.parserAppliedSeq).toBe(1) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(1) wsMocks.send.mockClear() act(() => { @@ -7261,7 +7261,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-opencode-heavy-replay', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() act(() => { rafCallbacks.shift()?.(16) @@ -7273,7 +7273,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-opencode-heavy-replay', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() expect(queryByText('Recovering terminal output...')).not.toBeNull() act(() => { @@ -7286,7 +7286,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-opencode-heavy-replay', - })?.parserAppliedSeq).toBe(chunks.length) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(chunks.length) }) it('does not checkpoint a stripped terminal.output.batch BEL segment as parser-applied', async () => { @@ -7321,10 +7321,15 @@ describe('TerminalView lifecycle updates', () => { }) expect(terminalWriteStrings(term)).toEqual(['A']) - expect(loadTerminalSurfaceCheckpoint(terminalId, { + // The strict APPLIED position never crosses the stripped BEL segment; + // the COVERAGE cursor does (a null-screen-effect completion signal) and + // is the resume position (responsive-terminal-restore WS2). + const strippedBelCheckpoint = loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-stripped-bel', - })?.parserAppliedSeq).toBe(1) + }, { paneId: 'pane-v2-stream' }) + expect(strippedBelCheckpoint?.parserAppliedSeq).toBe(1) + expect(strippedBelCheckpoint?.surfaceCoverageSeq).toBe(2) wsMocks.send.mockClear() act(() => { @@ -7334,7 +7339,7 @@ describe('TerminalView lifecycle updates', () => { expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.attach', terminalId, - sinceSeq: 1, + sinceSeq: 2, })) }) @@ -7391,10 +7396,15 @@ describe('TerminalView lifecycle updates', () => { await waitFor(() => { expect(queryByText('Recovering terminal output...')).toBeNull() }) - expect(loadTerminalSurfaceCheckpoint(terminalId, { + // No false APPLIED record (nothing rendered), but the completion + // signal IS consumed coverage: the resume position is 1, not a + // full-baseline rebuild (responsive-terminal-restore WS2). + const belOnlyCheckpoint = loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-replay-stripped-complete', - })).toBeNull() + }, { paneId: 'pane-v2-stream' }) + expect(belOnlyCheckpoint?.parserAppliedSeq).toBe(0) + expect(belOnlyCheckpoint?.surfaceCoverageSeq).toBe(1) wsMocks.send.mockClear() act(() => { @@ -7404,7 +7414,7 @@ describe('TerminalView lifecycle updates', () => { expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.attach', terminalId, - sinceSeq: 0, + sinceSeq: 1, })) }) @@ -7468,7 +7478,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-replay-stripped-tail', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() act(() => { delayedCallbacks[0]?.callback() @@ -7480,7 +7490,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-replay-stripped-tail', - })?.parserAppliedSeq).toBe(1) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(1) wsMocks.send.mockClear() act(() => { @@ -7528,7 +7538,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-batch-mixed-stripped-bel', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() wsMocks.send.mockClear() act(() => { @@ -7577,10 +7587,14 @@ describe('TerminalView lifecycle updates', () => { }) expect(terminalWriteStrings(term)).toEqual(['B']) - expect(loadTerminalSurfaceCheckpoint(terminalId, { + // No false APPLIED record (the BEL blocks the strict cursor at zero); + // the coverage cursor spans the filtered BEL and the applied tail. + const legacyStrippedCheckpoint = loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-legacy-stripped-bel', - })).toBeNull() + }, { paneId: 'pane-v2-stream' }) + expect(legacyStrippedCheckpoint?.parserAppliedSeq).toBe(0) + expect(legacyStrippedCheckpoint?.surfaceCoverageSeq).toBe(2) wsMocks.send.mockClear() act(() => { @@ -7590,7 +7604,7 @@ describe('TerminalView lifecycle updates', () => { expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.attach', terminalId, - sinceSeq: 0, + sinceSeq: 2, })) }) @@ -7642,10 +7656,12 @@ describe('TerminalView lifecycle updates', () => { await waitFor(() => { expect(queryByText('Recovering terminal output...')).toBeNull() }) - expect(loadTerminalSurfaceCheckpoint(terminalId, { + const legacyBelOnlyCheckpoint = loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-legacy-replay-stripped-complete', - })).toBeNull() + }, { paneId: 'pane-v2-stream' }) + expect(legacyBelOnlyCheckpoint?.parserAppliedSeq).toBe(0) + expect(legacyBelOnlyCheckpoint?.surfaceCoverageSeq).toBe(1) wsMocks.send.mockClear() act(() => { @@ -7655,7 +7671,7 @@ describe('TerminalView lifecycle updates', () => { expect(wsMocks.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'terminal.attach', terminalId, - sinceSeq: 0, + sinceSeq: 1, })) }) @@ -7722,7 +7738,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-legacy-replay-stripped-tail', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() act(() => { delayedCallbacks[0]?.callback() @@ -7734,7 +7750,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-legacy-replay-stripped-tail', - })?.parserAppliedSeq).toBe(1) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(1) wsMocks.send.mockClear() act(() => { @@ -7777,7 +7793,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId, serverInstanceId: 'server-output-legacy-mixed-stripped-bel', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() wsMocks.send.mockClear() act(() => { @@ -7840,7 +7856,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-active', serverInstanceId: 'server-missing-output-stream', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() wsMocks.send.mockClear() act(() => { @@ -7905,7 +7921,7 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-active', serverInstanceId: 'server-missing-gap-stream', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() wsMocks.send.mockClear() act(() => { @@ -8003,11 +8019,11 @@ describe('TerminalView lifecycle updates', () => { expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stored-stale-stream', serverInstanceId: 'server-missing-ready-stream', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: null, serverInstanceId: 'server-missing-ready-stream', - })).toBeNull() + }, { paneId: 'pane-v2-stream' })).toBeNull() wsMocks.send.mockClear() act(() => { @@ -8041,11 +8057,11 @@ describe('TerminalView lifecycle updates', () => { xtermVersion: '6.0.0', bufferType: 'unknown', parserIdle: true, - }) + }, { paneId: 'pane-v2-stream' }) expect(loadTerminalSurfaceCheckpoint(terminalId, { streamId: null, serverInstanceId, - })?.parserAppliedSeq).toBe(17) + }, { paneId: 'pane-v2-stream' })?.parserAppliedSeq).toBe(17) await renderTerminalHarness({ status: 'running', @@ -8207,7 +8223,7 @@ describe('TerminalView lifecycle updates', () => { const initialCheckpoint = loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-1', serverInstanceId: 'server-a', - }) + }, { paneId: 'pane-v2-stream' }) expect(initialCheckpoint?.attachRequestId).toBe(firstAttach?.attachRequestId) expect(initialCheckpoint?.parserAppliedSeq).toBe(1) @@ -8279,7 +8295,7 @@ describe('TerminalView lifecycle updates', () => { const checkpointAfterStaleCallback = loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-1', serverInstanceId: 'server-a', - }) + }, { paneId: 'pane-v2-stream' }) expect(checkpointAfterStaleCallback).not.toBeNull() expect(checkpointAfterStaleCallback?.attachRequestId).toBe(firstAttach?.attachRequestId) expect(checkpointAfterStaleCallback?.parserAppliedSeq).toBe(1) @@ -8309,7 +8325,7 @@ describe('TerminalView lifecycle updates', () => { const checkpointAfterCurrentCallback = loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-1', serverInstanceId: 'server-a', - }) + }, { paneId: 'pane-v2-stream' }) expect(checkpointAfterCurrentCallback?.attachRequestId).toBe(firstAttach?.attachRequestId) expect(checkpointAfterCurrentCallback?.parserAppliedSeq).toBe(1) }) @@ -8344,7 +8360,7 @@ describe('TerminalView lifecycle updates', () => { const initialCheckpoint = loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-delta', serverInstanceId: 'server-a', - }) + }, { paneId: 'pane-v2-stream' }) expect(initialCheckpoint?.attachRequestId).toBe(firstAttach?.attachRequestId) expect(initialCheckpoint?.parserAppliedSeq).toBe(1) @@ -8462,7 +8478,7 @@ describe('TerminalView lifecycle updates', () => { const checkpointAfterCallbacks = loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-delta', serverInstanceId: 'server-a', - }) + }, { paneId: 'pane-v2-stream' }) expect(checkpointAfterCallbacks?.attachRequestId).toBe(firstAttach?.attachRequestId) expect(checkpointAfterCallbacks?.parserAppliedSeq).toBe(1) @@ -8734,7 +8750,7 @@ describe('TerminalView lifecycle updates', () => { const trustedCheckpoint = loadTerminalSurfaceCheckpoint(terminalId, { streamId: 'stream-in-flight', serverInstanceId: 'server-a', - }) + }, { paneId: 'pane-v2-stream' }) expect(trustedCheckpoint?.parserAppliedSeq).toBe(1) const delayedCallbacks: Array<() => void> = [] @@ -9690,19 +9706,8 @@ describe('TerminalView lifecycle updates', () => { }) }) - it('recreates a restored OpenCode pane when visible viewport hydration cannot replay startup output', async () => { + it('keeps a restored OpenCode pane alive when visible viewport hydration cannot replay startup output (no auto-kill)', async () => { const sessionRef = { provider: 'opencode', sessionId: 'ses_focus_replay_gap' } as const - const addedRestoreIds = new Set() - restoreMocks.addTerminalRestoreRequestId.mockImplementation((id: string) => { - addedRestoreIds.add(id) - }) - restoreMocks.consumeTerminalRestoreRequestId.mockImplementation((id: string) => { - if (addedRestoreIds.has(id)) { - addedRestoreIds.delete(id) - return true - } - return false - }) const { store, tabId, paneId, terminalId, rerender } = await renderTerminalHarness({ status: 'running', @@ -9713,20 +9718,6 @@ describe('TerminalView lifecycle updates', () => { requestId: 'req-opencode-focus-gap', sessionRef, }) - // b8ke ext r20 F2: the restored session has an owner record — the - // replacement kill must carry its observed (epoch, generation) - // pair so a reconnect-queued stale kill is typed-refused instead - // of killing a newer owner. - store.dispatch(applyRuntimeOwner({ - type: 'session.runtimeOwner', - provider: 'opencode', - sessionId: 'ses_focus_replay_gap', - epoch: 12, - generation: 34, - ownerKind: 'terminal', - operationId: 'handoff-1', - transition: 'handoff-committed', - })) wsMocks.send.mockClear() @@ -9735,6 +9726,10 @@ describe('TerminalView lifecycle updates', () => {
)} + {deliveryGapNotice && ( + // Honest delivery-loss state (responsive-terminal-restore, + // round-5 finding 2): a negotiated queue_overflow / + // handoff_boundary_reached gap's accessible CHROME notice. Never a + // surface write — during the checkpoint delta repair NOTHING may + // mutate the xterm surface before the replayed bytes apply, so the + // notice rides React state and the screen below it is repaired to + // exactly the uninterrupted reference. +
+ {`Terminal output gap ${deliveryGapNotice.fromSeq}-${deliveryGapNotice.toSeq} (${deliveryGapNotice.reason}). The missing output is being refetched — the screen below is completed as it arrives.`} +
+ )} {recoveryExhausted && ( // Bounded automatic recovery (responsive-terminal-restore WS2): the // visible, accessible retry state. Automatic re-attach cycling was @@ -6968,7 +7021,7 @@ function TerminalView({ tabId, paneId, paneContent, hidden, focusEpoch = 0 }: Te
Terminal restore is not making progress. What is on screen is unchanged.