fix(hotblocks): drain on SIGTERM behind an unready window

Cargo.toml

@@ -47,6 +47,7 @@ serde_yaml = "0.9.33"
 scylla = "1.1.0"
 tempfile = "3.13.0"
 tokio = "1.42.0"
+tokio-util = "0.7.13"
 tracing = "0.1.41"
 tracing-subscriber = "0.3.19"
 url = "2.5.4"

crates/hotblocks-harness/src/sut.rs

@@ -59,6 +59,7 @@ pub struct Sut {
     dir: TempDir,
     port: u16,
     child: Option<Child>,
+    shutdown_started: Option<Instant>,
     /// How long the last boot took to accept connections (SLI-5).
     pub last_startup: Duration
 }
@@ -78,6 +79,7 @@ impl Sut {
             cfg,
             dir,
             child: None,
+            shutdown_started: None,
             last_startup: Duration::ZERO
         };
         sut.write_config()?;
@@ -116,32 +118,52 @@ impl Sut {
         };
         child.kill().await.context("failed to kill the SUT")?;
         self.child = None;
+        self.shutdown_started = None;
         Ok(())
     }
 
-    /// SIGTERM and wait for the drain-and-exit (LIV-12, `P-SHUTDOWN`).
-    pub async fn stop(&mut self) -> Result<ShutdownReport> {
-        let Some(mut child) = self.child.take() else {
+    /// Send SIGTERM without waiting, so a test can observe the drain sequence in progress.
+    pub fn signal_shutdown(&mut self) -> Result<()> {
+        let Some(child) = self.child.as_ref() else {
             bail!("the SUT is not running")
         };
         let pid = child.id().context("the SUT has no pid")? as i32;
-        let started = Instant::now();
+        ensure!(self.shutdown_started.is_none(), "shutdown was already requested");
 
         // SAFETY: `pid` names a child of this process that has not been reaped.
-        unsafe { libc::kill(pid, libc::SIGTERM) };
+        if unsafe { libc::kill(pid, libc::SIGTERM) } != 0 {
+            return Err(std::io::Error::last_os_error()).context("failed to send SIGTERM to the SUT");
+        }
+        self.shutdown_started = Some(Instant::now());
+        Ok(())
+    }
 
-        match tokio::time::timeout(Duration::from_secs(60), child.wait()).await {
+    /// Wait for a previously requested shutdown to finish.
+    pub async fn wait_for_shutdown(&mut self, timeout: Duration) -> Result<ShutdownReport> {
+        let Some(_) = self.child.as_ref() else {
+            bail!("the SUT is not running")
+        };
+        let started = self.shutdown_started.take().context("shutdown was not requested")?;
+        let mut child = self.child.take().expect("checked above");
+
+        match tokio::time::timeout(timeout, child.wait()).await {
             Ok(status) => Ok(ShutdownReport {
                 status: status.context("failed to wait for the SUT")?,
                 took: started.elapsed()
             }),
             Err(_) => {
                 child.kill().await.ok();
-                bail!("the SUT did not exit within 60s of SIGTERM")
+                bail!("the SUT did not exit within {timeout:?} of SIGTERM")
             }
         }
     }
 
+    /// SIGTERM and wait for the drain-and-exit (LIV-12, `P-SHUTDOWN`).
+    pub async fn stop(&mut self) -> Result<ShutdownReport> {
+        self.signal_shutdown()?;
+        self.wait_for_shutdown(Duration::from_secs(60)).await
+    }
+
     /// The tail of the service log, minus the HTTP access log — the harness's own polling makes
     /// that the loudest thing in there and never the interesting one.
     pub fn log_tail(&self, lines: usize) -> String {
@@ -184,6 +206,7 @@ impl Sut {
             .with_context(|| format!("failed to spawn {}", self.cfg.bin.display()))?;
 
         self.child = Some(child);
+        self.shutdown_started = None;
         self.await_ready(started).await?;
         self.last_startup = started.elapsed();
         Ok(())

crates/hotblocks/Cargo.toml

@@ -29,6 +29,7 @@ sqd-query = { path = "../query", features = ["storage"] }
 sqd-storage = { path = "../storage" }
 tikv-jemallocator = "0.6.0"
 tokio = { workspace = true, features = ["full"] }
+tokio-util = { workspace = true }
 tower-http = { version = "0.6.1", features = ["request-id", "trace"] }
 tracing = { workspace = true, features = ["valuable"] }
 tracing-subscriber = { workspace = true, features = ["env-filter", "json", "valuable"] }
@@ -39,7 +40,7 @@ anyhow = { workspace = true }
 reqwest = { workspace = true }
 sqd-hotblocks-harness = { path = "../hotblocks-harness" }
 tempfile = { workspace = true }
-tokio = { workspace = true, features = ["full"] }
+tokio = { workspace = true, features = ["full", "test-util"] }
 
 [lints]
 workspace = true

crates/hotblocks/spec/12-conformance-tdd.md

@@ -227,7 +227,7 @@ INV-21/22/23 (checks in parentheses):
 6. anchored continuation across responses never breaks parent-hash chains (INV-23);
 7. watermark coherence: `first ≤ fin ≤ head` whenever reported together (INV-5/30).
 
-## 5. Traceability matrix (status @ 2026-07-20)
+## 5. Traceability matrix (status @ 2026-07-21)
 
 Legend: **C** covered, **P** partial (some storage-layer or fixture coverage exists;
 service-level black-box coverage absent), **U** untested. Rows that changed with Phase 0 name
@@ -267,14 +267,14 @@ the same property under forks, crashes and retention is the business of CT-2/CT-
 | RP-19/20 lookup contract | CT-5 | **P** | hit/miss, disabled-index behavior, over-limit rejection before dataset lookup, and the 256-byte boundary are pinned; the NOT_FOUND vs UNKNOWN_DATASET split remains open (GAP-39) |
 | LIV-1/2 progress/stall | CT-6/7 | **U — known-violated** | GAP-1 |
 | LIV-3 query termination | CT-3/6 | U | |
-| LIV-4 waiter termination | CT-1/3 | P | `ct1_happy_path`: a query above the head answers `NO_DATA` within `P-HEAD-WAIT` |
+| LIV-4 waiter termination | CT-1/3 | **P — known-violated** | `ct1_happy_path`: a query above the head answers `NO_DATA` within `P-HEAD-WAIT`. The shutdown clause is unimplemented — the wait is a fixed timeout that never learns of the signal, so waiters are released by expiry rather than released *on* shutdown (GAP-17) |
 | LIV-5/6 startup/recovery | CT-2/6 | **U — known-violated** | GAP-7; `Sut::last_startup` records SLI-5 per boot |
 | LIV-7 reclamation | CT-7 | U | runtime reclaim on by default since 2026-07 (GAP-6 residual: boot-gated residue purge); convergence unmeasured |
 | LIV-8 isolation | CT-8 | **U — known-violated** | GAP-1/14 |
 | LIV-9 fork convergence/alarm | CT-4 | **U — known-violated** | GAP-5 now has a deterministic repro (`ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion`): one bad source of three parks ingestion — no alarm, no recovery |
 | LIV-10 overload recovery | CT-6 | U | |
 | LIV-11 retention keep-up | CT-7 | U | |
-| LIV-12 shutdown | CT-2 | **U — known-suspect** | GAP-17; `Sut::stop` measures the drain |
+| LIV-12 shutdown | CT-2 | **P — known-violated** | `ct2_shutdown` drives the real binary through SIGTERM and pins unready-before-grace, continued serving during grace, the hard drain deadline with an in-flight long-poll, bounded exit and success status. The clean-truncation and ingest-wind-down halves are still open and the deployed grace still forces SIGKILL (GAP-17) |
 | RS-3/4 window floor/excess | CT-1/7 | U | |
 | RS-6 amplification | CT-7 | U | reclaim path fixed 2026-07 (GAP-6); bound unmeasured under churn |
 | RS-8 boot maintenance | CT-2/7 | P | unlink/orphan-purge behaviors have storage-level tests |
@@ -288,7 +288,7 @@ the same property under forks, crashes and retention is the business of CT-2/CT-
 | OB-2..11 | all | P | query metrics exist; stall gauges pending on PR #83 (unmerged); OB-2 heartbeat, OB-6 debt accounting, OB-9 alarms, OB-11 forensics absent |
 | OB-12 index state | CT-1 | **P** | CF-wide estimated keys / live SST bytes are exported for both indexes; per-dataset enabled/count/bytes and lookup hit/miss/latency remain absent (GAP-40) |
 
-## 6. Gap register (dated 2026-07-20, informative)
+## 6. Gap register (dated 2026-07-21, informative)
 
 Known or strongly suspected divergences between this spec and the current system, from
 incident history, code-level review, and coverage analysis. Priorities: P0 = active
@@ -312,7 +312,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po
 | GAP-14 | Read-side capacity (execution slots, waiter slots) is a single global pool: one dataset's query herd can starve all datasets | PF-4, LIV-8 | P2 | CT-8: herd on D′, tip-follower SLOs on D |
 | GAP-15 | No explicit store-format compatibility gate at boot; incompatibility surfaces as runtime decode errors | CN-12, INV-43 | P3 | CT-5 boot matrix with future-format fixture |
 | GAP-16 | ~~The service layer has essentially zero automated tests~~ — **closed by Phase 0**, see §6.1 | all | — | done |
-| GAP-17 | Shutdown can take a panic-class exit path in ingestion cancellation (observed at redeploy) | LIV-12, FM-PROC-4 | P2 | CT-2 shutdown class: SIGTERM under load ×100, zero panic exits |
+| GAP-17 | Shutdown can take a panic-class exit path in ingestion cancellation (observed at redeploy). **2026-07-21 addendum**: replica replacement is client-visible, in two ways that must not be conflated. (a) *Idle pooled connection* — at SIGTERM the server closes keep-alive connections at once while the endpoint removal is still propagating (chart: grace 5 s, no `preStop`, no readiness probe at all), so the portal POSTs into a socket already gone: 502 burst observed one second after SIGTERM with a Ready sibling idle. Nothing was admitted, so LIV-12 does permit it, and masking it belongs outside this spec — portal replay on a fresh connection (`sqd-portal` 13-conformance GAP-22) plus a chart `preStop`. (b) *Active stream cut* — **not** permitted: RP-15 requires the truncation to be "observationally identical to a budget stop", and a process kill leaves an unterminated chunked body and a half-written codec frame, so the client cannot decode the prefix at all, let alone as valid JSONL (IB-5's looser paraphrase is not a licence). A response that has already committed 200 and emitted bytes is unreplayable by any client, so (b) is fixable only here. And (b) is not rare: the drain floor already equals `P-HEAD-WAIT`, because head-waiters run a fixed 5 s timeout with no knowledge of the signal (`query/service.rs`) in violation of LIV-4's explicit shutdown clause — that is the whole 5 s grace, so under tip-follower load the exit is SIGKILL, i.e. FM-PROC-1, and the "clean path" this entry once assumed is not taken in production at all. The drain is also unbounded from the inside: `axum::serve(..).with_graceful_shutdown` carries no deadline and GAP-29 (no response deadline; a stalled reader pins its response) means nothing bounds `P-SHUTDOWN` whatever the grace is. Lastly the RP-15 escape hatch owes a truncation counter that does not exist (GAP-10). **Partly closed 2026-07-21**: SIGTERM now runs the same two-phase sequence as the portal (sqd-portal#113) — `/ready` reports 503 for `--pre-drain-grace-secs` while everything else serves normally, then the drain runs under a hard `--drain-timeout-secs`; SIGINT stays on the default handler so dev Ctrl-C is unaffected. That answers (a) and bounds `P-SHUTDOWN` from the inside, but only takes effect once the chart probes `/ready` and raises the grace above the sum of the two. Still open: (b), since a stream cut at the deadline is still a reset rather than an RP-15 end; LIV-4's shutdown clause, since waiters still sit out their fixed timeout; and the original ingest-cancellation path | LIV-12, LIV-4, RP-15, FM-PROC-4 | **P1** (was P2 on the "bounded or rare" premise; the addendum removes it — it fires on every replica replacement and the clean path is never taken under load) | CT-2 shutdown class: SIGTERM under load ×100 — zero panic exits, exit ≤ `P-SHUTDOWN`, every cut stream decodes as a valid JSONL prefix, every long-poll released ≤ `P-HEAD-WAIT` |
 | GAP-18 | Dual-writer detection exists only on some paths (finality/head updates), not all mutations | WP-15, FM-OP-3 | P3 | CT-5: two harness-driven writers, assert loser stops on every mutation type |
 | GAP-20 | `parent_number` linkage is never validated on any layer (the block trait exposes it; nothing reads it): a hash-linked run can claim an arbitrarily higher number for the next block, storing a false hole on a densely-numbered chain — a silent data gap served as if it were a slot gap. (The originally-filed non-monotonic-numbers scenario is unreachable today: the source-position advance forces ascending numbers.) | WP-2, DEF-4, INV-1 | P2 | CT-4/CT-9: hash-linked run with a number jump on a dense chain; the run MUST be rejected with no state change |
 | GAP-21 | An anchored query whose `from` sits mid-chunk above a number gap larger than the conflict-check lookback (a hard-coded 100 positions in the plan's base-block check) fails `INTERNAL` instead of evaluating the assertion. A >100-position hole with an anchor landing just above it is probably unrealistic, hence low priority. A correct all-predecessors scan was tried and reverted 2026-07-15 — it regressed `check_parent_block` into an unbounded per-chunk scan+sort; needs a lazy sort-desc + limit(100) | RP-11, INV-26 | P3 | CT-5 `ct5_anchor_is_evaluated_across_a_large_number_hole` (`#[ignore]` until fixed): >100-position hole in one chunk; anchored query just above must yield OK/CONFLICT, never 500 |
@@ -358,7 +358,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po
 
   | Built, awaiting scripts | Missing |
   |---|---|
-  | `Sut::crash/stop/restart` (same db, same port) → CT-2 | the CT-2 kill-point matrix |
+  | `Sut::crash/stop/restart` (same db, same port); `ct2_shutdown` exercises the bounded SIGTERM path | the remaining CT-2 kill-point matrix |
   | `Harness::fork` + `Model::resolve_fork` + the follower's CONFLICT recovery → CT-4 | the CT-4 fork/finality corpus |
   | `Model::predict_query` + initial `ct5_error_soundness` matrix | remaining CT-5 binding, boot, and overload rows |
   | `SimFaults` injection point → CT-9 | the rest of the FM-SRC repertoire |
@@ -370,10 +370,12 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po
 
 - **Phase 1 — the P0s.** Stall harness + OB-2/3/11 signals (GAP-1); churn soak + space
   accounting via OB-6 (GAP-6). These target the two production incidents.
-- **Phase 2 — correctness core.** CT-2 crash matrix (GAP-2), CT-4 fork/finality corpus
-  (GAP-3/4/5), CT-5 remaining error taxonomy + boot matrix (GAP-8/9/15/36/39).
+- **Phase 2 — correctness core.** CT-2 crash matrix (GAP-2) and shutdown class (GAP-17 —
+  pulled forward from Phase 3: it is client-visible on every replica replacement), CT-4
+  fork/finality corpus (GAP-3/4/5), CT-5 remaining error taxonomy + boot matrix
+  (GAP-8/9/15/36/39).
 - **Phase 3 — robustness.** CT-9 fuzz both surfaces (GAP-12), CT-3 concurrency swarms,
-  CT-8 isolation (GAP-14), shutdown class (GAP-17).
+  CT-8 isolation (GAP-14).
 - **Phase 4 — performance regime.** CT-6 scenarios S1–S6, SLO gates, saturation knees,
   baselines (GAP-7/13); wire into CI with tolerances.
 

crates/hotblocks/spec/13-interface-binding.md

@@ -34,6 +34,7 @@ annotated with the relevant GAP.
 | GET-RETENTION | `GET /datasets/{id}/retention` | current policy JSON |
 | SET-RETENTION | `POST /datasets/{id}/retention` | policy JSON; only for `External` datasets, else `FORBIDDEN` (403) |
 | observability | `GET /metrics` (+ engine-diagnostic routes) | OB surface, text formats |
+| readiness | `GET /ready` | rotation gate (OB-8), distinct from the `/` liveness signal: 503 for the whole pre-drain grace window so the orchestrator withdraws the endpoint before anything closes (LIV-12). Process-level only — per-dataset readability (LIV-5c) is still absent (GAP-7) |
 
 Dialects accepted in query bodies: `evm`, `solana`, `bitcoin`, `tron`,
 `hyperliquidFills`, `hyperliquidReplicaCmds` — and, expressible in the query schema but
@@ -71,8 +72,13 @@ below" never applies below genesis).
   projected, and anchored continuation (RP-10) therefore requires projecting them.
 - **IB-5** The success status is committed before streaming begins; admission-time errors
   therefore arrive as proper error statuses, while post-admission failures surface as
-  RP-15 truncation (stream ends early; already-sent prefix remains valid JSONL). See
-  GAP-10 for the observability obligation.
+  RP-15 truncation (stream ends early; already-sent prefix remains valid JSONL). At this
+  binding a truncation is a *server-side end*, not a dropped connection: the codec stream
+  and the chunked body MUST both be terminated, so that the client sees a short but
+  well-framed response, exactly as at a budget stop (RP-15). A reset connection leaves a
+  half-written codec frame the client cannot decode at all — a transport failure, not a
+  truncation, and not licensed by this clause (GAP-17). See GAP-10 for the observability
+  obligation.
 - **IB-6** Response watermark headers (success): finalized head number+hash when defined,
   and a head-number header (for the live stream: at least the snapshot head; may be
   fresher per CN-5). `NO_DATA` (204) SHOULD carry the same watermark headers — currently

crates/hotblocks/spec/14-parameters.md

@@ -62,7 +62,7 @@ ran against.
 | `P-ALARM` | integrity fault → observable alarm (WP-17, LIV-9b, OB-9) | ∞ (no alarm states exist: GAP-5) | ≤ 10 s ⚠ |
 | `P-STARTUP-ACCEPT` | process start → accepting connections (LIV-5a, SLI-5) | ~35 s observed (GAP-7) | ≤ 3 s ⚠ |
 | `P-STARTUP-READY(state)` | per-dataset readable bound (LIV-5b, SLI-6) | — | budget curve vs state size ⚠ |
-| `P-SHUTDOWN` | drain-and-exit bound (LIV-12) | — | ≤ 30 s ⚠ |
+| `P-SHUTDOWN` | drain-and-exit bound (LIV-12) | `--pre-drain-grace-secs` 25 s + `--drain-timeout-secs` 25 s since 2026-07-21 (GAP-17), bounded in-process at last; the deployment's 5 s grace is still below the sum, so the exit remains SIGKILL until the chart is raised | deployment grace ≥ `pre_drain_grace + drain_timeout`; drain deadline ≥ `P-QUERY-TIME` + `P-SCHED-SLACK` ⚠ |
 | `P-DUR-PROCESS` | commits lost on process crash (CN-6) | 0 | 0 |
 | `P-DUR-SYSTEM` | commit-suffix loss window on host/power failure (CN-6b) | bounded, engine-managed (not explicitly configured) | make explicit ⚠ |
 | `P-QUIESCENCE` | harness settling period before model comparison (12 §1) | — | 2× `P-CLEANUP-PERIOD` ⚠ |