From 89069a747081117d9c57688a93bbb8f46edd9d6c Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Mon, 20 Jul 2026 14:39:15 +0400 Subject: [PATCH] fix(hotblocks): keep ingesting when a block time is unrepresentable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The datetime conversion in `flush` feeds nothing but a `debug!` field, yet its `?` killed the batch and wedged the dataset on a 60 s retry loop forever — at info level, over a line that is never emitted. `time` is informational (DEF-4, CN-8), so the conversion is best-effort now and the raw millis are logged beside it. The seconds→millis conversions saturate so an absurd source value neither panics in debug nor wraps into a plausible date. Co-Authored-By: Claude Opus 4.8 --- crates/data/src/evm/model.rs | 3 ++- crates/data/src/solana/model.rs | 3 ++- crates/hotblocks/spec/12-conformance-tdd.md | 4 ++-- .../hotblocks/src/dataset_controller/ingest_generic.rs | 9 +++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/data/src/evm/model.rs b/crates/data/src/evm/model.rs index dc575998..5224dabe 100644 --- a/crates/data/src/evm/model.rs +++ b/crates/data/src/evm/model.rs @@ -344,7 +344,8 @@ impl sqd_primitives::Block for Block { } fn timestamp(&self) -> Option { - Some(self.header.timestamp * 1000) + // saturate: an absurd source value must not panic (debug) or wrap into a plausible date + Some(self.header.timestamp.saturating_mul(1000)) } fn data_availability_mask(&self) -> DataMask { diff --git a/crates/data/src/solana/model.rs b/crates/data/src/solana/model.rs index ff129023..81920013 100644 --- a/crates/data/src/solana/model.rs +++ b/crates/data/src/solana/model.rs @@ -203,6 +203,7 @@ impl sqd_primitives::Block for Block { } fn timestamp(&self) -> Option { - self.header.timestamp.map(|seconds| seconds * 1000) + // saturate: an absurd source value must not panic (debug) or wrap into a plausible date + self.header.timestamp.map(|seconds| seconds.saturating_mul(1000)) } } diff --git a/crates/hotblocks/spec/12-conformance-tdd.md b/crates/hotblocks/spec/12-conformance-tdd.md index 49441293..388da896 100644 --- a/crates/hotblocks/spec/12-conformance-tdd.md +++ b/crates/hotblocks/spec/12-conformance-tdd.md @@ -279,7 +279,7 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | RS-6 amplification | CT-7 | U | reclaim path fixed 2026-07 (GAP-6); bound unmeasured under churn | | RS-8 boot maintenance | CT-2/7 | P | unlink/orphan-purge behaviors have storage-level tests | | RS-10/11 residue/deletion cost | CT-7 | P | GAP-6/13 | -| FM-1 robustness | CT-9 | **P — known-violated** | GAP-12/26 open; unsupported query and query-worker panic classes are closed (§6.1), and the unterminated-record class is pinned by `ct9_source_faults` | +| FM-1 robustness | CT-9 | **P — known-violated** | GAP-12 open; unsupported query and query-worker panic classes are closed (§6.1), and the unterminated-record class is pinned by `ct9_source_faults` | | FM-SRC-* corpus | CT-4 | U | one stale-pack crash-loop already occurred (GAP-5 class); no strike/quarantine substrate (GAP-30) | | FM-STOR-2/3 disk pressure | CT-7 | U | incident-derived; no automated test | | FM-OP-1..5 | CT-5 | U | | @@ -320,7 +320,6 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-23 | Window trims drop the anchor hash: the automatic trim passes no hash and the retained state stores `⊥`, though the correct value sits unused in the first batch's `parent_block_hash`. Disables below-window divergence detection (WP-6b has nothing to contradict) and feeds GAP-22 | INV-18, DEF-7, WP-6b | P1 | CT-1: CONFLICT hints / STATUS at the window edge after a trim; CT-4: below-window fork after a trim must RESET, not absorb silently | | GAP-24 | One dataset's init failure aborts the whole service: startup propagates the first controller error (kind mismatch, retention bail, corrupt state) instead of alarming that dataset and serving the rest | CN-10, FM-OP-1, INV-36, INV-43 | P1 | CT-5 boot matrix: corrupt one dataset's persisted state; assert the others serve and the broken one alarms | | GAP-25 | Downward retention (`from < first(D)`) executes as an *implicit, unobservable* RESET (WP §2.5 as amended 2026-07-12 legalizes the destruction, but requires OB-9 observability) — no event, indistinguishable from a trim; and the boot-time `Pinned` equivalent aborts the entire service (via GAP-24) instead of a dataset-level refusal | WP §2.5, OB-9, INV-43 | P2 | CT-1: SET-RETENTION below `first`; assert a RESET observable + serving continuity; CT-5: boot with lowered `Pinned.from` | -| GAP-26 | A block timestamp outside the datetime-conversion range kills the ingest flush — the conversion exists only for a log line — and the dataset enters a permanent crash-loop on redelivery | FM-1, WP-18, CN-8 | P1 | CT-9: serve a block with `time = i64::MAX`; the dataset must ingest it and keep serving | | GAP-27 | FINALIZE never checks `e ≥ first(D)`: with `fin = ⊥` (e.g. after a trim passed above it) a lagging source's report below the window commits `fin < first(D)` | WP §2.4, INV-5 | P2 | CT-4: finality below the window on a trimmed dataset; assert the report is ignored | | GAP-28 | The External retention instruction and the empty-dataset anchor are memory-only (the persisted label holds kind/version/fin) — a restart forgets the instructed bound (the dataset re-idles awaiting a new instruction) and resets an empty dataset's anchor | CN-9, INV-40, WP-11 | P2 | CT-2: SET-RETENTION, restart; assert the bound and anchor are recovered | | GAP-29 | A stalled-but-open connection pins the response's storage snapshot indefinitely (no overall response deadline / server write timeout; only disconnects release it); pinned snapshots block physical reclaim of point-deleted data | CN-7, RP-18, HZ-9, RS-6 | P2 | CT-3/CT-7: zombie client that stops reading; assert snapshot lifetime ≤ `P-QUERY-TIME` and reclaim proceeds | @@ -341,6 +340,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-11 | Unsupported `substrate` / `fuel` queries and query-worker panics escaped the HTTP error taxonomy by panicking their request task | Typed `UNSUPPORTED_QUERY` admission errors, panic containment in the query executor, CT-5 dialect requests, and an executor unit test (2026-07-15) | | GAP-16 | No service-level automated tests | [`crates/hotblocks-harness`](../../hotblocks-harness) + `ct1_happy_path` (Phase 0, 2026-07-12) | | GAP-19 | A source response whose final JSONL record carried no trailing newline panicked the line reader (`LineStream::take_final_line` left its scan position past the emptied buffer). The ingest task died, its buffered batch was lost, and the dataset parked for `P-EPOCH-RETRY` — then crash-looped, since the source served the same body on retry. Violated FM-1, LIV-2 | Found by CT-1 on the harness's first run; fixed in `crates/data-client/src/reqwest/lines.rs`; pinned by a unit test there and by `ct9_source_faults` (2026-07-12) | +| GAP-26 | A block timestamp outside the datetime-conversion range killed the ingest flush — the conversion existed only for a log line | Log formatting is best-effort and the raw value is stored unchanged; evm/solana seconds→millis saturate so an absurd source value neither panics nor wraps (PR #100, 2026-07-20). No regression test — CT-9: serve a block with `time = i64::MAX`; assert the batch commits and serving continues | | GAP-32 | A finalized-head trim/reset race could turn an admitted finalized query into `INTERNAL` when its snapshot no longer had a finalized head | Snapshot-time absence now maps to `NO_DATA`; pinned by `finalized_snapshot_without_a_head_is_no_data` (2026-07-15) | | GAP-38 | `TX-BY-HASH` / `tidx` absent; fork re-inclusion ordering unimplemented | Transaction hash CF + independent flag + HTTP binding; storage transition suite and black-box ingest/reorg/re-inclusion tests (2026-07-15) | diff --git a/crates/hotblocks/src/dataset_controller/ingest_generic.rs b/crates/hotblocks/src/dataset_controller/ingest_generic.rs index ebff077e..c4046986 100644 --- a/crates/hotblocks/src/dataset_controller/ingest_generic.rs +++ b/crates/hotblocks/src/dataset_controller/ingest_generic.rs @@ -3,7 +3,7 @@ use std::{ time::Instant }; -use anyhow::{anyhow, ensure}; +use anyhow::ensure; use chrono::{DateTime, Utc}; use futures::StreamExt; use sqd_data_core::{BlockChunkBuilder, ChunkProcessor, PreparedChunk}; @@ -250,14 +250,15 @@ where let last_block = self.last_block; let last_block_hash = self.last_block_hash.clone(); - let last_block_time = DateTime::::from_timestamp_millis(self.last_block_time.unwrap_or(0)) - .ok_or_else(|| anyhow!("block time is out of range"))?; + // informational only (DEF-4, CN-8): an unrepresentable value must not fail the flush + let last_block_time = DateTime::::from_timestamp_millis(self.last_block_time.unwrap_or(0)); debug!( first_block = first_block, last_block = last_block, last_block_hash = %last_block_hash, - last_block_time = %last_block_time.format("%Y-%m-%dT%H:%M:%S%.3fZ"), + last_block_time = ?last_block_time, + last_block_time_ms = ?self.last_block_time, finalized_head = valuable(&self.finalized_head), "received new chunk" );