From e0aa8d31d6610e66838cb5996237ec7b91ea67e4 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 1 Jul 2026 20:10:09 +0100 Subject: [PATCH 01/22] fix(client): let store AIMD ramp off the cold-start floor (V2-554) On fast links the store adaptive-concurrency limiter sat pinned at its cold-start floor of ~8 for a whole upload, capping throughput. Three compounding causes, all fixed here: 1. Transport quorum-shortfall pollution. A close-group PUT that misses quorum only because dead/stale relayed peer addresses could not be dialled surfaced as `InsufficientPeers` -> `NetworkError`, counting against `success_rate` and cutting the cap. This is remote peer churn (same root cause as V2-551), not local backpressure. Split per-peer failures into timeout (genuine backpressure) vs dial/relay churn, and route the aggregate: any timeout keeps `InsufficientPeers` (still cuts the cap), a pure dial-churn shortfall surfaces the new neutral `Error::CloseGroupShortfall` (classified `ApplicationError`), and an app-only shortfall keeps its representative error (V2-468). The new variant stays a recoverable/deferred shortfall in the merkle retry. 2. Asymmetric increase/decrease gate. A Decrease fires every `min_window_ops` (8) samples but an Increase only every `window_ops` (32), and a Decrease zeroed the increase counter, so a few-percent shortfall permanently starved growth. Retain the increase credit across a Decrease AND relax the store `success_target` 0.95 -> 0.88 so normal per-chunk noise reads as a healthy window; genuine congestion (larger shortfall / timeout ceiling) still cuts the cap. Store-scoped; quote keeps the classic gate. 3. Per-wave/per-file cap snapshot. Store fan-out used `buffer_unordered(cap)` read once per wave/file, so mid-flight limiter growth never reached in-flight work. `merkle_store_with_retry` and the wave `store_paid_chunks_with_events` now roll a `FuturesUnordered` that re-reads the cap as each slot frees (the byte budget still bounds the wave path). Covers the merkle wave, whole-file, and deferred-retry paths. All logic is in ant-core with unit tests for each fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-core/src/data/client/adaptive.rs | 131 ++++++++++++++++++++++- ant-core/src/data/client/batch.rs | 69 +++++++----- ant-core/src/data/client/chunk.rs | 127 ++++++++++++++++------- ant-core/src/data/client/file.rs | 11 +- ant-core/src/data/client/merkle.rs | 150 +++++++++++++++++++-------- ant-core/src/data/client/mod.rs | 25 ++++- ant-core/src/data/error.rs | 26 +++++ 7 files changed, 420 insertions(+), 119 deletions(-) diff --git a/ant-core/src/data/client/adaptive.rs b/ant-core/src/data/client/adaptive.rs index ab586799..cc4e5153 100644 --- a/ant-core/src/data/client/adaptive.rs +++ b/ant-core/src/data/client/adaptive.rs @@ -379,6 +379,23 @@ pub struct LimiterConfig { /// `Ok(None)` (Timeout) rate, which the timeout_ceiling check /// catches. pub latency_decrease_enabled: bool, + /// When `true`, a Decrease does NOT reset `samples_since_increase`, so + /// evidence already accrued toward the next Increase survives a transient + /// Decrease instead of being zeroed. + /// + /// The default (`false`) reproduces the original asymmetric gate: an + /// Increase needs a full fresh `window_ops` of samples, a Decrease needs + /// only `min_window_ops`, AND a Decrease resets the increase counter. On a + /// channel where decreases fire faster than a full increase window can + /// accrue, that combination permanently starves growth — the store cap sat + /// pinned at its cold-start floor of 8 for a whole 530-chunk upload + /// (V2-554). The store channel sets this `true` so a few-percent shortfall + /// rate can no longer deny the cap its next doubling: the accrued increase + /// credit persists across the transient Decrease, and the next genuinely + /// healthy window (`evaluate` returns Increase) can act on it. Sustained + /// unhealthiness still holds the cap down because `evaluate` keeps + /// returning Decrease — this only stops isolated dips from zeroing progress. + pub retain_increase_credit_on_decrease: bool, } impl LimiterConfig { @@ -393,10 +410,11 @@ impl LimiterConfig { timeout_ceiling: cfg.timeout_ceiling, latency_inflation_factor: cfg.latency_inflation_factor, latency_ewma_alpha: cfg.latency_ewma_alpha, - // Defaults preserve the original AIMD behaviour; the fetch - // channel overrides both in `AdaptiveController::new`. + // Defaults preserve the original AIMD behaviour; the fetch and + // store channels override these in `AdaptiveController::new`. slow_start_ramp_threshold: 0, latency_decrease_enabled: true, + retain_increase_credit_on_decrease: false, } } @@ -725,7 +743,13 @@ fn apply_decision(inner: &mut LimiterInner, decision: Decision, cfg: &LimiterCon debug!(from = inner.current, to = next, "adaptive: decrease"); } inner.current = next; - inner.samples_since_increase = 0; + // Retaining the increase credit (store channel, V2-554) keeps the + // evidence accrued toward the next Increase alive across a transient + // Decrease so a few-percent shortfall rate can't permanently starve + // growth. The default zeroes it, requiring a full fresh window. + if !cfg.retain_increase_credit_on_decrease { + inner.samples_since_increase = 0; + } inner.samples_since_decrease = 0; } Decision::Hold => {} @@ -1085,6 +1109,28 @@ impl AdaptiveController { // to a crawl. See the fetch override and `LimiterConfig` field docs. store_cfg.latency_decrease_enabled = false; store_cfg.slow_start_ramp_threshold = usize::MAX; + // Break the asymmetric gate that pinned the store cap at its cold-start + // floor (V2-554). Two coupled changes, both scoped to store: + // + // 1. Retain the increase credit across a Decrease. The default gate + // needs a full fresh `window_ops` of samples to earn an Increase but + // only `min_window_ops` to fire a Decrease, AND a Decrease zeroes the + // increase counter — so an isolated dip discards accrued growth. + // Retaining it lets the next healthy window act on that evidence. + // + // 2. Relax `success_target` from the global 0.95 to 0.88. Retaining the + // credit is not enough on its own: a Decrease still fires every + // `min_window_ops` (8) samples while eligible but an Increase only + // every `window_ops` (32), a 4:1 firing ratio that lets a sustained + // few-percent shortfall outpace growth and crush the cap regardless. + // At 0.95 just 2 shortfalls in a 32-op window (0.9375) trip a + // Decrease — normal per-chunk close-group noise. At 0.88 a Decrease + // needs ~4 shortfalls in 32 (~12%), so a few-percent shortfall reads + // as a healthy window and the cap can ramp. Genuine congestion (a + // larger shortfall rate, or the unchanged `timeout_ceiling`) still + // cuts the cap. Quote keeps the classic 0.95 gate. + store_cfg.retain_increase_credit_on_decrease = true; + store_cfg.success_target = 0.88; let mut fetch_cfg = LimiterConfig::from_adaptive(&config, config.max.fetch); // Lift the fetch channel's floor above the global // `min_concurrency`. Reasoning is specific to download: on @@ -1610,6 +1656,7 @@ mod tests { latency_ewma_alpha: 0.5, slow_start_ramp_threshold: 0, latency_decrease_enabled: true, + retain_increase_credit_on_decrease: false, } } @@ -1869,6 +1916,23 @@ mod tests { c.quote.config.slow_start_ramp_threshold, 0, "quote must keep classic AIMD slow-start exit", ); + assert!( + !c.quote.config.retain_increase_credit_on_decrease, + "quote must keep the classic gate (Decrease resets the increase counter)", + ); + assert!( + c.store.config.retain_increase_credit_on_decrease, + "store must retain increase credit across a Decrease (V2-554)", + ); + assert!( + (c.store.config.success_target - 0.88).abs() < f64::EPSILON, + "store must relax success_target to 0.88 so a few-percent shortfall still ramps (V2-554), got {}", + c.store.config.success_target, + ); + assert!( + (c.quote.config.success_target - c.config().success_target).abs() < f64::EPSILON, + "quote must keep the global success_target", + ); // Store now mirrors fetch on these two knobs: node-side merkle // verification latency is not local congestion, and a transient // Decrease must not condemn the cap to a +1-per-window crawl. @@ -1981,6 +2045,67 @@ mod tests { ); } + #[test] + fn store_gate_rebalance_ramps_where_classic_gate_pins() { + // V2-554 gate rebalance, end-to-end. Two limiters driven by the SAME + // borderline sequence — a steady ~5% shortfall (1 error per 20 + // successes), the normal per-chunk close-group noise floor: + // + // - `fixed`: the store tuning (success_target 0.88 + retain increase + // credit). A ~5% shortfall reads as a healthy window, so it ramps. + // - `classic`: the pre-V2-554 store gate (success_target 0.95 + reset + // on Decrease). Just 2 shortfalls in a 32-op window (0.9375) trip a + // Decrease that fires 4x faster than an Increase can be earned and + // zeroes the increase credit, so the cap stays pinned at the floor. + let build = |success_target: f64, retain: bool| { + let mut cfg = cfg_for_tests(); + cfg.min_concurrency = 1; + cfg.max_concurrency = 256; + cfg.window_ops = 32; + cfg.min_window_ops = 8; + cfg.success_target = success_target; + cfg.timeout_ceiling = 0.10; + // Both stay slow-start-armed with no latency-decrease so the only + // difference under test is the gate (target + retain). + cfg.slow_start_ramp_threshold = usize::MAX; + cfg.latency_decrease_enabled = false; + cfg.retain_increase_credit_on_decrease = retain; + Limiter::new(8, cfg) + }; + let fixed = build(0.88, true); + let classic = build(0.95, false); + + for _ in 0..80 { + for _ in 0..20 { + fixed.observe(Outcome::Success, Duration::from_millis(5)); + classic.observe(Outcome::Success, Duration::from_millis(5)); + } + fixed.observe(Outcome::NetworkError, Duration::from_millis(5)); + classic.observe(Outcome::NetworkError, Duration::from_millis(5)); + } + + // The rebalanced gate ramps well off the cold-start floor of 8 under + // the noise floor that pinned the classic gate. + assert!( + fixed.current() >= 32, + "rebalanced store gate must ramp off the floor under ~5% shortfall, got {}", + fixed.current(), + ); + // The classic gate stays crushed at/near the floor on the same input — + // this is the V2-554 pathology. + assert!( + classic.current() <= 8, + "classic gate should stay pinned near the floor on the same input, got {}", + classic.current(), + ); + assert!( + fixed.current() > classic.current(), + "rebalanced gate {} must out-grow the classic gate {}", + fixed.current(), + classic.current(), + ); + } + #[test] fn cold_start_clamps_into_bounds() { let cfg = cfg_for_tests(); diff --git a/ant-core/src/data/client/batch.rs b/ant-core/src/data/client/batch.rs index ef3b0397..d277f939 100644 --- a/ant-core/src/data/client/batch.rs +++ b/ant-core/src/data/client/batch.rs @@ -20,7 +20,7 @@ use ant_protocol::payment::{ use ant_protocol::transport::{MultiAddr, PeerId}; use ant_protocol::{compute_address, XorName, DATA_TYPE_CHUNK}; use bytes::Bytes; -use futures::stream::{self, StreamExt}; +use futures::stream::{self, FuturesUnordered, StreamExt}; use std::collections::{HashMap, HashSet}; use std::time::{Duration, Instant}; use tokio::sync::mpsc; @@ -784,35 +784,48 @@ impl Client { chunk_attempts_total = chunk_attempts_total.saturating_add(to_retry.len()); let store_limiter = self.controller().store.clone(); - let store_concurrency = store_limiter - .current() - .min(to_retry.len().max(1)) - .min(byte_bound); - let mut upload_stream = stream::iter(to_retry) - .map(|chunk| { - let chunk_clone = chunk.clone(); - let limiter = store_limiter.clone(); - async move { - let result = observe_op( - &limiter, - || async move { - self.chunk_put_to_close_group( - chunk.content, - chunk.proof_bytes, - &chunk.quoted_peers, - ) - .await - }, - classify_error, - ) - .await; - (chunk_clone, result) - } - }) - .buffer_unordered(store_concurrency); + // Rolling scheduler: keep up to `cap()` stores in flight and re-read + // the cap as each slot frees, so mid-flight limiter growth reaches + // the rest of this wave instead of being frozen at a per-wave + // snapshot (V2-554). The in-flight BYTE budget (`byte_bound`) stays + // enforced so a wave of large chunks can't OOM a small host; + // iterator exhaustion bounds launches to the wave, so no explicit + // clamp to `to_retry.len()` is needed. + let make_store = |chunk: PaidChunk| { + let chunk_clone = chunk.clone(); + let limiter = store_limiter.clone(); + async move { + let result = observe_op( + &limiter, + || async move { + self.chunk_put_to_close_group( + chunk.content, + chunk.proof_bytes, + &chunk.quoted_peers, + ) + .await + }, + classify_error, + ) + .await; + (chunk_clone, result) + } + }; + let mut chunk_iter = to_retry.into_iter(); + let mut in_flight = FuturesUnordered::new(); let mut failed_this_round = Vec::new(); - while let Some((chunk, result)) = upload_stream.next().await { + loop { + let slots = store_limiter.current().min(byte_bound).max(1); + while in_flight.len() < slots { + match chunk_iter.next() { + Some(chunk) => in_flight.push(make_store(chunk)), + None => break, + } + } + let Some((chunk, result)) = in_flight.next().await else { + break; + }; match result { Ok(name) => { let duration_ms = first_seen diff --git a/ant-core/src/data/client/chunk.rs b/ant-core/src/data/client/chunk.rs index 8e33b5cc..4f9fd338 100644 --- a/ant-core/src/data/client/chunk.rs +++ b/ant-core/src/data/client/chunk.rs @@ -25,9 +25,11 @@ use tracing::{debug, info, warn}; const CHUNK_DATA_TYPE: u32 = 0; /// Why a single-peer PUT was declined. Drives the surfaced aggregate error -/// and keeps the store AIMD limiter honest — only a transport shortfall is a -/// "client is sending too fast" signal (V2-468); a node that responds with a -/// structured rejection is an application-level decline (ADR-0002). +/// and keeps the store AIMD limiter honest — only genuine local backpressure +/// (a PUT-response `Timeout`) is a "client is sending too fast" signal; a node +/// that responds with a structured rejection is an application-level decline +/// (ADR-0002 / V2-468), and a bare dial/relay failure is remote peer churn, +/// not local capacity (V2-554). #[derive(Clone, Copy)] enum PutRejection { /// Node is out of storage (`ProtocolError::StorageFailed`) — try a @@ -41,13 +43,24 @@ enum PutRejection { PriceFloor, /// Some other structured remote rejection. OtherRemote, - /// Transport/timeout failure — the node did not respond. - Transport, + /// The peer accepted the connection but did not answer the PUT within the + /// deadline (`Error::Timeout`). This is the genuine local-backpressure + /// signal: under real congestion the client's own requests time out, so a + /// shortfall carrying any timeout stays a capacity signal (V2-554). + Timeout, + /// A dial/relay/transport failure — the peer could not be reached at all + /// (`Error::Network` and other non-response errors), typically a dead or + /// stale relayed DHT address. This is remote peer churn, not local + /// backpressure, so a shortfall made up purely of these must not push the + /// store AIMD limiter down (V2-554). + Dial, } -/// Classify a failed single-peer PUT (ADR-0002). A `RemotePut` carries the -/// node's structured `ProtocolError`; a `PaymentRequired` response surfaces as -/// [`Error::Payment`]; anything else is a transport failure. +/// Classify a failed single-peer PUT (ADR-0002 / V2-468 / V2-554). A +/// `RemotePut` carries the node's structured `ProtocolError`; a +/// `PaymentRequired` response surfaces as [`Error::Payment`]; a +/// [`Error::Timeout`] is genuine local backpressure; anything else is a +/// dial/relay failure (remote churn). fn classify_put_failure(error: &Error) -> PutRejection { match error { Error::RemotePut { source, .. } => match source { @@ -61,30 +74,47 @@ fn classify_put_failure(error: &Error) -> PutRejection { // price-floor `PaymentFailed` — not a transport shortfall, so it must // not push the store AIMD limiter down (ADR-0002 / V2-468). Error::Payment(_) => PutRejection::PriceFloor, - _ => PutRejection::Transport, + // The peer did not answer in time: genuine local backpressure. + Error::Timeout(_) => PutRejection::Timeout, + // Could not reach the peer at all: dial/relay churn (remote), not a + // local-capacity signal. + _ => PutRejection::Dial, } } /// Decide the error for a close-group store that fell short of quorum. /// -/// When every failure was an application-level decline — the node responded -/// (full / price-floor / `PaymentRequired` / other remote rejection) and there -/// was **no** transport failure — return the representative application error so -/// the shortfall classifies as `ApplicationError` and does not push the store -/// AIMD limiter down as a false capacity signal (ADR-0002 / V2-468). A shortfall -/// that included any transport failure is a genuine capacity signal and surfaces -/// as `InsufficientPeers` (classified `NetworkError`). +/// Only genuine local backpressure should push the store AIMD limiter down. +/// The failure mix decides which signal to surface: +/// +/// 1. **Any PUT-response timeout** (`timeout > 0`): the client's own requests +/// are timing out — genuine local congestion. Surface `InsufficientPeers` +/// (classified `NetworkError`) so the limiter still backs off (V2-554). +/// 2. **No timeouts, no dial failures** — every failure was an application +/// decline (full / price-floor / `PaymentRequired` / other remote +/// rejection): surface the representative application error so the shortfall +/// classifies `ApplicationError` and does not suppress the limiter +/// (ADR-0002 / V2-468). +/// 3. **No timeouts, but dial/relay failures present**: the shortfall is +/// close-group dial churn (dead/stale relayed peer addresses) — remote peer +/// churn, not local capacity. Surface [`Error::CloseGroupShortfall`] +/// (classified `ApplicationError`) so it does NOT push the limiter down +/// (V2-554). Still recoverable/retryable. fn put_shortfall_error( - transport: usize, + timeout: usize, + dial: usize, first_app_rejection: Option, - insufficient_peers_message: String, + shortfall_message: String, ) -> Error { - if transport == 0 { + if timeout > 0 { + return Error::InsufficientPeers(shortfall_message); + } + if dial == 0 { if let Some(app_rejection) = first_app_rejection { return app_rejection; } } - Error::InsufficientPeers(insufficient_peers_message) + Error::CloseGroupShortfall(shortfall_message) } /// Result of one sweep over a chunk's close group. @@ -405,7 +435,8 @@ impl Client { let mut full = 0usize; let mut price_floor = 0usize; let mut other_remote = 0usize; - let mut transport = 0usize; + let mut timeout = 0usize; + let mut dial = 0usize; let mut first_app_rejection: Option = None; while let Some((peer_id, result)) = put_futures.next().await { @@ -427,7 +458,8 @@ impl Client { PutRejection::Full => full += 1, PutRejection::PriceFloor => price_floor += 1, PutRejection::OtherRemote => other_remote += 1, - PutRejection::Transport => transport += 1, + PutRejection::Timeout => timeout += 1, + PutRejection::Dial => dial += 1, } // An application-level decline is `RemotePut` (a structured // node rejection) or `Error::Payment` (`PaymentRequired`): @@ -458,17 +490,19 @@ impl Client { } } - // Quorum not reached. An application-only shortfall surfaces the - // representative app error (so it doesn't suppress the limiter); a - // shortfall with any transport failure is a real capacity signal. + // Quorum not reached. A timeout-bearing shortfall is genuine local + // backpressure (capacity signal); an application-only shortfall surfaces + // the representative app error; a pure dial-churn shortfall surfaces a + // neutral `CloseGroupShortfall` (V2-554). See `put_shortfall_error`. let aggregate = format!( "Stored on {success_count} peers, need {CLOSE_GROUP_MAJORITY} \ (full: {full}, price-floor: {price_floor}, other-rejection: {other_remote}, \ - transport: {transport}). Failures: [{}]", + timeout: {timeout}, dial: {dial}). Failures: [{}]", failures.join("; ") ); Err(put_shortfall_error( - transport, + timeout, + dial, first_app_rejection, aggregate, )) @@ -1031,7 +1065,7 @@ mod tests { const TEST_DISTANCE_TAIL_INDEX: usize = TEST_XORNAME_BYTE_LEN - 1; #[test] - fn classify_put_failure_maps_remote_and_transport_reasons() { + fn classify_put_failure_maps_remote_timeout_and_dial_reasons() { let remote = |source| Error::RemotePut { address: "test-addr".to_string(), source, @@ -1056,34 +1090,51 @@ mod tests { classify_put_failure(&Error::Payment("Payment required: more".to_string())), PutRejection::PriceFloor )); + // A PUT-response timeout is genuine local backpressure (V2-554). assert!(matches!( classify_put_failure(&Error::Timeout("no response".to_string())), - PutRejection::Transport + PutRejection::Timeout + )); + // A dial/relay failure (dead/stale relayed address) is remote churn. + assert!(matches!( + classify_put_failure(&Error::Network("dial failed".to_string())), + PutRejection::Dial )); } #[test] - fn put_shortfall_surfaces_app_error_only_without_transport_failure() { + fn put_shortfall_routes_by_failure_mix() { let app = || Error::Payment("Payment required: more".to_string()); let msg = || "shortfall".to_string(); - // Every failure was an application-level decline (e.g. all peers asked - // for more payment): surface the app error so the limiter isn't driven - // down as a false capacity signal (ADR-0002 / V2-468). + // Every failure was an application-level decline (no timeout, no dial): + // surface the app error so the limiter isn't driven down as a false + // capacity signal (ADR-0002 / V2-468). assert!(matches!( - put_shortfall_error(0, Some(app()), msg()), + put_shortfall_error(0, 0, Some(app()), msg()), Error::Payment(_) )); - // A transport failure in the mix: a genuine capacity shortfall. + // Any PUT-response timeout in the mix is genuine local backpressure: + // keep it a capacity signal so the store limiter still backs off (V2-554). assert!(matches!( - put_shortfall_error(1, Some(app()), msg()), + put_shortfall_error(1, 0, Some(app()), msg()), Error::InsufficientPeers(_) )); - // No application rejection captured at all (pure transport): capacity. assert!(matches!( - put_shortfall_error(0, None, msg()), + put_shortfall_error(1, 3, None, msg()), Error::InsufficientPeers(_) )); + // No timeouts but dial/relay churn present: remote peer churn, not local + // capacity — surface a neutral CloseGroupShortfall (V2-554). + assert!(matches!( + put_shortfall_error(0, 2, None, msg()), + Error::CloseGroupShortfall(_) + )); + // Dial churn alongside an app rejection, still no timeout: neutral. + assert!(matches!( + put_shortfall_error(0, 1, Some(app()), msg()), + Error::CloseGroupShortfall(_) + )); } fn chunk_peer_get_result(peer_seed: u8, distance_tail: u8) -> ChunkPeerGetResult { diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index 12d0ddc3..fac483d3 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -2598,9 +2598,12 @@ impl Client { wave.len() ); - // Clamp fan-out to wave size — partial last wave should - // not pay for extra slots (see PERF-RESULTS.md). - let store_concurrency = store_limiter.current().min(wave.len().max(1)); + // Cap closure re-read per scheduler refill so mid-flight limiter + // growth reaches the rest of the wave (V2-554). Clamped to wave size + // — a partial last wave should not pay for extra slots (see + // PERF-RESULTS.md). + let wave_len = wave.len(); + let cap = || store_limiter.current().min(wave_len.max(1)); let chunks: Vec<([u8; 32], Bytes)> = wave .into_iter() .map(|(content, addr)| (addr, content)) @@ -2614,7 +2617,7 @@ impl Client { // across waves. let outcome = merkle_store_with_retry( chunks, - store_concurrency, + cap, 1, std::time::Duration::ZERO, progress, diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index 7957c4f6..dce8cb63 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -835,7 +835,10 @@ impl Client { addresses.len() ))); } - let store_concurrency = store_limiter.current().min(batch_size.max(1)); + // Cap closure re-read per scheduler refill so mid-flight limiter growth + // is applied to the rest of the batch (V2-554). Clamped to batch size — + // partial batches should not pay for unused slots (see PERF-RESULTS.md). + let cap = || store_limiter.current().min(batch_size.max(1)); let chunks: Vec<([u8; 32], Bytes)> = addresses.into_iter().zip(chunk_contents).collect(); @@ -867,7 +870,7 @@ impl Client { let outcome = merkle_store_with_retry( chunks, - store_concurrency, + cap, MERKLE_STORE_MAX_ATTEMPTS, MERKLE_RETRY_BACKOFF, progress, @@ -947,15 +950,18 @@ pub(crate) struct MerkleStoreOutcome { /// Drive a set of merkle chunk stores with bounded retry of quorum shortfalls. /// -/// Runs `store_one` over all `chunks` concurrently (up to `store_concurrency`), -/// collecting any `InsufficientPeers` failures rather than aborting. Failed -/// chunks are retried — `store_one` re-collects their close group on each call, -/// so a converged routing table can yield a fresh group — for up to -/// `max_attempts` rounds, sleeping a jittered `backoff` between rounds. A -/// chunk's success is counted once and recorded in the retry round it landed on -/// (`retries_histogram[round]`). `stored_offset` seeds the returned `stored` -/// count and the progress numbering; `total` is the whole-file total reported -/// in progress events. +/// Runs `store_one` over all `chunks` concurrently, keeping up to `cap()` stores +/// in flight and RE-READING `cap()` as each slot frees — so mid-flight adaptive +/// growth is applied to the rest of the round instead of being frozen at a +/// per-round snapshot (V2-554). Collects quorum shortfalls +/// (`InsufficientPeers`, `CloseGroupShortfall`, `RemotePut`) rather than +/// aborting. Failed chunks are retried — `store_one` re-collects their close +/// group on each call, so a converged routing table can yield a fresh group — +/// for up to `max_attempts` rounds, sleeping a jittered `backoff` between +/// rounds. A chunk's success is counted once and recorded in the retry round it +/// landed on (`retries_histogram[round]`). `stored_offset` seeds the returned +/// `stored` count and the progress numbering; `total` is the whole-file total +/// reported in progress events. /// /// A non-quorum error stops the pass but does **not** discard progress: the /// successes already completed this pass stay in `stored`/`stored_addresses`, @@ -964,9 +970,9 @@ pub(crate) struct MerkleStoreOutcome { /// Callers that want the old abort-everything behaviour re-raise `fatal` as /// `Err`; CLI callers fold it into `PartialUpload` while keeping the stores. #[allow(clippy::too_many_arguments)] -pub(crate) async fn merkle_store_with_retry( +pub(crate) async fn merkle_store_with_retry( chunks: Vec<([u8; 32], Bytes)>, - store_concurrency: usize, + cap: C, max_attempts: usize, backoff: Duration, progress: Option<&mpsc::Sender>, @@ -977,6 +983,7 @@ pub(crate) async fn merkle_store_with_retry( where F: Fn([u8; 32], Bytes) -> Fut, Fut: std::future::Future>, + C: Fn() -> usize, { let attempts = max_attempts.max(1); let mut outcome = MerkleStoreOutcome { @@ -986,19 +993,31 @@ where let mut pending = chunks; for attempt in 0..attempts { - let concurrency = store_concurrency.min(pending.len().max(1)).max(1); // Carries the chunk body forward for the next round plus the last // quorum-shortfall message, so an exhausted set can report per-chunk // errors via `failed_addresses`. let mut next_failed: Vec<([u8; 32], Bytes, String)> = Vec::new(); - let mut upload_stream = stream::iter(pending.into_iter().map(|(addr, content)| { - let fut = store_one(addr, content.clone()); - async move { (addr, content, fut.await) } - })) - .buffer_unordered(concurrency); - - while let Some((addr, content, result)) = upload_stream.next().await { + // Rolling scheduler: keep up to `cap()` stores in flight, re-reading the + // cap as each slot frees so limiter growth mid-round is applied to the + // remaining chunks (V2-554). Iterator exhaustion bounds the launch count + // to the pending set, so no explicit clamp to `pending.len()` is needed. + let mut pending_iter = pending.into_iter(); + let mut in_flight = FuturesUnordered::new(); + loop { + let slots = cap().max(1); + while in_flight.len() < slots { + match pending_iter.next() { + Some((addr, content)) => { + let fut = store_one(addr, content.clone()); + in_flight.push(async move { (addr, content, fut.await) }); + } + None => break, + } + } + let Some((addr, content, result)) = in_flight.next().await else { + break; + }; outcome.stats.chunk_attempts_total = outcome.stats.chunk_attempts_total.saturating_add(1); match result { @@ -1018,12 +1037,17 @@ where }); } } - // A quorum shortfall — whether reported as a transport - // shortfall (`InsufficientPeers`) or an app-only rejection + // A quorum shortfall — whether a timeout-bearing capacity + // shortfall (`InsufficientPeers`), a pure dial-churn shortfall + // (`CloseGroupShortfall`, V2-554), or an app-only rejection // (`RemotePut`, e.g. pool-rejected / quote-stale / disk-full, // which are transient) — is recoverable: defer and retry the - // chunk rather than aborting the whole upload (V2-468). - Err(e @ (Error::InsufficientPeers(_) | Error::RemotePut { .. })) => { + // chunk rather than aborting the whole upload (V2-468 / V2-554). + Err( + e @ (Error::InsufficientPeers(_) + | Error::CloseGroupShortfall(_) + | Error::RemotePut { .. }), + ) => { next_failed.push((addr, content, e.to_string())); } Err(e) => { @@ -1211,11 +1235,13 @@ where // resident at a time), so the deferred path keeps the wave path's // memory bound regardless of how many chunks were deferred. let chunks = read_bodies(&batch_addrs)?; - let concurrency = concurrency_for(batch_addrs.len()); + // Re-read the cap per scheduler refill (V2-554) via `concurrency_for`, + // which re-samples the store limiter clamped to this batch's size. + let cap = || concurrency_for(batch_addrs.len()); let batch_outcome = merkle_store_with_retry( chunks, - concurrency, + cap, 1, Duration::ZERO, progress, @@ -1798,9 +1824,10 @@ mod tests { } }; - let outcome = merkle_store_with_retry(chunks, 8, 1, Duration::ZERO, None, 0, 6, store_one) - .await - .expect("quorum shortfalls must not abort the batch"); + let outcome = + merkle_store_with_retry(chunks, || 8, 1, Duration::ZERO, None, 0, 6, store_one) + .await + .expect("quorum shortfalls must not abort the batch"); assert_eq!(outcome.stored, 4); assert_eq!(outcome.failed, 2); @@ -1809,6 +1836,38 @@ mod tests { assert_eq!(outcome.stats.chunk_attempts_total, 6); } + /// V2-554: the store scheduler must RE-READ the cap as each slot frees + /// (rolling), not snapshot it once like `buffer_unordered`. A snapshot would + /// invoke the cap closure once per attempt; the rolling scheduler invokes it + /// once per drained slot, so mid-flight limiter growth reaches the rest of + /// the round. Proven here by counting cap-closure invocations. + #[tokio::test] + async fn store_with_retry_rereads_cap_per_slot() { + let count = 6; + let chunks = make_chunks(count); + let cap_calls = Arc::new(Mutex::new(0usize)); + let cap_calls_for_closure = cap_calls.clone(); + let cap = move || { + *cap_calls_for_closure.lock().expect("cap counter poisoned") += 1; + 2 + }; + let store_one = + move |_addr: [u8; 32], _content: Bytes| async move { Ok(std::time::Instant::now()) }; + + let outcome = + merkle_store_with_retry(chunks, cap, 1, Duration::ZERO, None, 0, count, store_one) + .await + .expect("all stores succeed"); + + assert_eq!(outcome.stored, count); + let calls = *cap_calls.lock().expect("cap counter poisoned"); + assert!( + calls >= count, + "cap must be re-read per drained slot (rolling), not snapshotted once — \ + expected >= {count} invocations, got {calls}", + ); + } + /// V2-468: an app-only quorum shortfall surfaces as `Error::RemotePut` /// (pool-rejected / quote-stale / disk-full — transient), which must be /// treated as recoverable just like `InsufficientPeers`: collected and @@ -1836,9 +1895,10 @@ mod tests { } }; - let outcome = merkle_store_with_retry(chunks, 8, 1, Duration::ZERO, None, 0, 6, store_one) - .await - .expect("remote app-rejections must not abort the batch"); + let outcome = + merkle_store_with_retry(chunks, || 8, 1, Duration::ZERO, None, 0, 6, store_one) + .await + .expect("remote app-rejections must not abort the batch"); assert_eq!(outcome.stored, 4); assert_eq!(outcome.failed, 2); @@ -1854,9 +1914,10 @@ mod tests { Err::(Error::Payment("missing proof".into())) }; - let outcome = merkle_store_with_retry(chunks, 8, 3, Duration::ZERO, None, 0, 3, store_one) - .await - .expect("fatal is carried in the outcome, not returned as Err"); + let outcome = + merkle_store_with_retry(chunks, || 8, 3, Duration::ZERO, None, 0, 3, store_one) + .await + .expect("fatal is carried in the outcome, not returned as Err"); assert!(matches!(outcome.fatal, Some(Error::Payment(_)))); } @@ -1876,9 +1937,10 @@ mod tests { } }; - let outcome = merkle_store_with_retry(chunks, 1, 1, Duration::ZERO, None, 0, 6, store_one) - .await - .expect("fatal carried in outcome, not returned as Err"); + let outcome = + merkle_store_with_retry(chunks, || 1, 1, Duration::ZERO, None, 0, 6, store_one) + .await + .expect("fatal carried in outcome, not returned as Err"); assert!(matches!(outcome.fatal, Some(Error::Payment(_)))); // The five chunks stored before the abort are preserved, not lost. assert_eq!(outcome.stored, 5); @@ -1917,7 +1979,7 @@ mod tests { }; let outcome = - merkle_store_with_retry(chunks, 8, 3, Duration::ZERO, None, 0, total, store_one) + merkle_store_with_retry(chunks, || 8, 3, Duration::ZERO, None, 0, total, store_one) .await .expect("should converge after retry"); @@ -1964,7 +2026,7 @@ mod tests { }; let outcome = - merkle_store_with_retry(chunks, 8, 3, Duration::ZERO, None, 0, total, store_one) + merkle_store_with_retry(chunks, || 8, 3, Duration::ZERO, None, 0, total, store_one) .await .expect("flaky chunk should recover on retry"); @@ -1992,7 +2054,7 @@ mod tests { let outcome = merkle_store_with_retry( chunks, - 8, + || 8, MERKLE_STORE_MAX_ATTEMPTS, Duration::ZERO, None, @@ -2038,7 +2100,7 @@ mod tests { let outcome = merkle_store_with_retry( chunks, - 8, + || 8, MERKLE_STORE_MAX_ATTEMPTS, Duration::ZERO, None, @@ -2073,7 +2135,7 @@ mod tests { let outcome = merkle_store_with_retry( chunks, - 8, + || 8, MERKLE_STORE_MAX_ATTEMPTS, Duration::ZERO, None, diff --git a/ant-core/src/data/client/mod.rs b/ant-core/src/data/client/mod.rs index 5e54009a..ddd085ad 100644 --- a/ant-core/src/data/client/mod.rs +++ b/ant-core/src/data/client/mod.rs @@ -63,6 +63,11 @@ pub(crate) const PUT_TARGET_WIDTH: usize = 20; /// - `RemotePut` -> `ApplicationError` (the remote node responded with a /// structured rejection — the transport succeeded, so the node declined /// at the application layer; not a local capacity signal) +/// - `CloseGroupShortfall` -> `ApplicationError` (a quorum shortfall caused +/// by close-group dial/relay churn with no PUT-response timeouts — remote +/// peer churn, not local backpressure; a timeout-bearing shortfall keeps +/// `InsufficientPeers`/`NetworkError` instead, so genuine congestion still +/// cuts the cap — V2-554) pub(crate) fn classify_error(err: &Error) -> Outcome { match err { Error::Timeout(_) => Outcome::Timeout, @@ -88,7 +93,14 @@ pub(crate) fn classify_error(err: &Error) -> Outcome { // transport round-trip succeeded, so the node declined at the // application layer (payment/disk/quote/pool). Not a local // capacity signal; recorded but must not push the limiter down. - | Error::RemotePut { .. } => Outcome::ApplicationError, + | Error::RemotePut { .. } + // A close-group PUT shortfall caused purely by dial/relay churn + // (dead/stale relayed peer addresses), with no PUT-response + // timeouts to signal local backpressure. Remote peer churn, not + // "client sending too fast" — must not push the limiter down + // (V2-554). A shortfall that DID time out keeps `InsufficientPeers` + // (`NetworkError`) so real congestion still cuts the cap. + | Error::CloseGroupShortfall(_) => Outcome::ApplicationError, } } @@ -669,6 +681,14 @@ mod tests { }, Outcome::ApplicationError, ), + // A close-group quorum shortfall caused by dial/relay churn with + // no PUT-response timeouts — remote peer churn, not local + // backpressure, so it must NOT register as a capacity signal + // (V2-554). A timeout-bearing shortfall keeps `InsufficientPeers`. + ( + Error::CloseGroupShortfall("Stored on 3 peers, need 4".to_string()), + Outcome::ApplicationError, + ), ]; for (err, expected) in &cases { let got = classify_error(err); @@ -750,7 +770,8 @@ mod tests { | Error::Cancelled(_) | Error::PartialUpload { .. } | Error::BadQuoteBinding { .. } - | Error::RemotePut { .. } => (), + | Error::RemotePut { .. } + | Error::CloseGroupShortfall(_) => (), }; } } diff --git a/ant-core/src/data/error.rs b/ant-core/src/data/error.rs index e3ad8706..f6e94386 100644 --- a/ant-core/src/data/error.rs +++ b/ant-core/src/data/error.rs @@ -43,6 +43,23 @@ pub enum Error { source: ant_protocol::ProtocolError, }, + /// A chunk PUT missed its close-group quorum, and the shortfall was + /// caused by close-group **dial/relay churn** — dead or stale relayed + /// peer addresses that could not be dialled — with **no** evidence of + /// local backpressure (no PUT-response timeouts among the failures). + /// + /// This is remote peer churn (the same dead relayed DHT addresses as + /// V2-551), NOT evidence the client is sending too fast. More local send + /// concurrency neither causes nor fixes it, so it classifies as + /// `Outcome::ApplicationError` (see `classify_error`) and does NOT push + /// the adaptive store limiter down (V2-554). Distinct from + /// [`Error::InsufficientPeers`], which a close-group shortfall keeps when + /// any PUT-response *timeout* is present (genuine local backpressure that + /// must still cut the cap). Like `InsufficientPeers`, it is a recoverable + /// quorum shortfall and is deferred/retried, not fatal. + #[error("close-group PUT shortfall (dial churn): {0}")] + CloseGroupShortfall(String), + /// Invalid data received. #[error("invalid data: {0}")] InvalidData(String), @@ -237,6 +254,15 @@ mod tests { assert_eq!(err.to_string(), "insufficient peers: need 5, got 2"); } + #[test] + fn test_display_close_group_shortfall() { + let err = Error::CloseGroupShortfall("Stored on 3 peers, need 4".to_string()); + assert_eq!( + err.to_string(), + "close-group PUT shortfall (dial churn): Stored on 3 peers, need 4" + ); + } + #[test] fn test_display_signature_verification() { let err = Error::SignatureVerification("invalid sig".to_string()); From c37afd935b13deca00b0fea30c82a1eec1970a72 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 2 Jul 2026 00:12:17 +0100 Subject: [PATCH 02/22] perf(client): de-wave the merkle store phase into one cap-bounded fan-out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole-file merkle upload stored chunks in sequential fixed 64-chunk waves with a hard barrier between them. On a churny network the wave barrier is the throughput bottleneck: a handful of chunks whose close-group peers are dead/stale relayed addresses take minutes to dial/revalidate, and the whole wave cannot advance until its slowest chunk finishes — so the next wave's chunks don't even begin peer resolution. A production 2.36 GB / 609-chunk upload spent ~70% of its wall-clock in these inter-wave gaps (measured on STG-UL-01), while the V2-554 store limiter had ramped to 64 and the actual store bursts hit ~50 concurrent. Store concurrency was no longer the long pole; the wave serialization was. Remove the waves. The payment/proofs are already computed up front, so the store phase only needs a memory bound, not batching. Store the whole file as a single cap-bounded fan-out: - `merkle_store_with_retry` now takes addresses and a `store_one(addr)` that acquires its own body (reads it from the spill on demand), so only the <=cap in-flight stores hold a body. The store cap maxes at 64, so peak resident memory stays ~64 x MAX_CHUNK_SIZE — the same bound the 64-chunk waves gave — with no barrier. A slow straggler now overlaps with the rest of the file instead of stalling it. - `merkle_deferred_retry` drops its per-batch `read_bodies`/`batch_size` memory-bounding (the cap now bounds memory) and stores each round's whole pending set in one pass. - `upload_merkle_from_spill` (renamed from `upload_waves_merkle`) makes a single store call over all chunks instead of looping waves; deferred retry, stats, progress, and fatal/PartialUpload handling are preserved. New tests: the store pass has no barrier (a blocked straggler must not deadlock the others) and holds at most `cap` bodies in flight (memory bound). This overlaps but does not fix the underlying dead-relay resolution latency (V2-551); it stops that latency from serializing. Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-core/src/data/client/file.rs | 277 ++++++++---------- ant-core/src/data/client/merkle.rs | 448 +++++++++++++++-------------- 2 files changed, 344 insertions(+), 381 deletions(-) diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index fac483d3..1bf94b07 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -592,16 +592,6 @@ impl ChunkSpill { Ok(Bytes::from(data)) } - /// Read a wave of chunks from disk. - fn read_wave(&self, wave_addrs: &[[u8; 32]]) -> Result> { - let mut out = Vec::with_capacity(wave_addrs.len()); - for addr in wave_addrs { - let content = self.read_chunk(addr)?; - out.push((content, *addr)); - } - Ok(out) - } - /// Clean up the spill directory. fn cleanup(&self) { if let Err(e) = std::fs::remove_dir_all(&self.dir) { @@ -634,7 +624,7 @@ fn cached_merkle_covers_addresses( /// A partial [`MerkleBatchPaymentResult`] (from a `pay_for_merkle_multi_batch` /// where a later sub-batch's payment failed) carries proofs only for the /// already-paid sub-batches, so unpaid chunks reach the upload path with no -/// proof. `upload_waves_merkle` reports those as failed via +/// proof. `upload_merkle_from_spill` reports those as failed via /// [`Error::PartialUpload`] rather than aborting the whole file. Order within /// each group follows `addresses`. fn partition_addresses_by_proof( @@ -716,7 +706,7 @@ struct SingleWaveOutcome { /// A `PartialUpload` (chunks short of quorum after retries) is **recoverable**: /// its stored/failed chunks and on-chain spend are returned so the caller /// records them and continues to the next wave, making the file make maximum -/// progress exactly like `upload_waves_merkle`. Every other error is **fatal** +/// progress exactly like `upload_merkle_from_spill`. Every other error is **fatal** /// (wallet/payment-infrastructure failures, missing proofs, spill reads) and is /// returned via `Err` to abort the file. Because `UPLOAD_WAVE_SIZE == /// PAYMENT_WAVE_SIZE`, each batch call is exactly one payment wave, so folding a @@ -2047,7 +2037,7 @@ impl Client { resuming with cached merkle proofs" ); let (stored, sc, gc, stats) = self - .upload_waves_merkle( + .upload_merkle_from_spill( &spill, &spill.addresses, cached, @@ -2125,7 +2115,7 @@ impl Client { reusing cached merkle proofs" ); let (stored, sc, gc, stats) = self - .upload_waves_merkle( + .upload_merkle_from_spill( &spill, &merkle_plan.to_upload, cached, @@ -2240,7 +2230,7 @@ impl Client { }; let (stored, sc, gc, stats) = self - .upload_waves_merkle( + .upload_merkle_from_spill( &spill, &merkle_plan.to_upload, &batch_result, @@ -2365,7 +2355,7 @@ impl Client { // A wave whose chunks fall short of quorum after retries must not abort // the file: its failures are accumulated here and surfaced as a single // `PartialUpload` only after every wave has been attempted, mirroring - // `upload_waves_merkle`. Aborting on the first failed wave (the old `?`) + // `upload_merkle_from_spill`. Aborting on the first failed wave (the old `?`) // discarded all later waves' progress — already self-encrypted, spilled, // and in some cases already paid for — converting high per-chunk success // into 0% per-file success. @@ -2409,7 +2399,7 @@ impl Client { // Fold this wave's result. A quorum shortfall (`PartialUpload`) is // recoverable and its parts are returned to be recorded here; // genuinely fatal errors propagate via `?` and abort the file, as in - // `upload_waves_merkle`. + // `upload_merkle_from_spill`. let outcome = fold_single_wave( self.batch_upload_chunks_with_events( wave_data, @@ -2494,20 +2484,23 @@ impl Client { /// Upload chunks from a spill using pre-computed merkle proofs. /// - /// Reads one wave at a time from disk, pairs each chunk with its proof, - /// and uploads concurrently. Peak memory: ~`UPLOAD_WAVE_SIZE × MAX_CHUNK_SIZE`. - /// - /// A chunk that is transiently short of quorum (`InsufficientPeers`) does - /// **not** abort the file, nor does it block the pipeline: each wave is - /// stored in a **single pass** (no in-wave backoff barrier), and chunks - /// short of quorum are collected into a file-level deferred set rather than - /// retried in place. After the last wave, [`merkle_deferred_retry`] retries - /// the whole deferred set in concurrent rounds ([`DEFERRED_ROUND_DELAYS_SECS`] - /// delays), re-reading each chunk's body from the spill and reusing its - /// proof. This keeps every wave running at full fan-out instead of parking - /// idle slots behind one slow chunk's backoff, while peak memory stays - /// bounded (bodies are re-read from disk, never pinned). Non-quorum errors - /// (e.g. a missing proof) stay fatal and abort immediately. + /// Stores the whole file as a **single cap-bounded fan-out** — not in fixed + /// waves. The store concurrency limiter is the only throttle: `store_one` + /// reads each chunk's body from the on-disk spill on demand, so at most + /// `store_cap` (≤ 64) bodies are ever resident, giving the same + /// `~store_cap × MAX_CHUNK_SIZE` peak-memory bound the old 64-chunk waves + /// gave — but with **no wave barrier**, so a slow straggler (e.g. a chunk + /// whose close-group peers are stale relayed addresses that take minutes to + /// revalidate) no longer stalls the rest of the file behind it. + /// + /// A chunk that is transiently short of quorum (`InsufficientPeers` / + /// `CloseGroupShortfall` / `RemotePut`) does **not** abort the file, nor + /// block the pass: the store pass is a **single attempt** (no in-pass + /// backoff), and quorum-short chunks are collected into a deferred set. After + /// the pass, [`merkle_deferred_retry`] retries that set in concurrent rounds + /// ([`DEFERRED_ROUND_DELAYS_SECS`] delays), re-reading each body from the + /// spill and reusing its proof. Non-quorum errors (e.g. a missing proof) + /// stay fatal and abort immediately. /// /// Returns `(chunks_stored, storage_cost_atto, gas_cost_wei)` on success. /// Costs come from the `batch_result` which was populated during payment. @@ -2515,9 +2508,9 @@ impl Client { /// # Errors /// /// Returns [`Error::PartialUpload`] if any chunk is still short of quorum - /// after all retries across every wave (other chunks remain stored), or the - /// underlying error for a non-quorum failure. - async fn upload_waves_merkle( + /// after the store pass and every deferred round (other chunks remain + /// stored), or the underlying error for a non-quorum failure. + async fn upload_merkle_from_spill( &self, spill: &ChunkSpill, addresses: &[[u8; 32]], @@ -2529,10 +2522,6 @@ impl Client { let total_chunks = total_stored + addresses.len(); let mut stored_addresses: Vec<[u8; 32]> = already_stored_addresses.to_vec(); let mut failed: Vec<([u8; 32], String)> = Vec::new(); - // Chunks short of quorum on their single wave pass are collected here and - // retried after the last wave (see `merkle_deferred_retry`), so a slow - // chunk never holds its wave's other slots idle behind a backoff. - let mut deferred: Vec<([u8; 32], String)> = Vec::new(); let mut agg_stats = WaveAggregateStats::default(); // Chunks without a merkle proof were never paid for: a partial @@ -2557,17 +2546,20 @@ impl Client { } } - let waves: Vec<&[[u8; 32]]> = to_store.chunks(UPLOAD_WAVE_SIZE).collect(); - let wave_count = waves.len(); - let store_limiter = self.controller().store.clone(); // Store one chunk to its (freshly re-collected) close group, reusing the - // chunk's merkle proof. Shared across every retry round so a converged - // routing table yields a fresh group. Only `InsufficientPeers` is - // recoverable; a missing proof stays fatal. Mirrors the external-signer - // path's closure in `merkle_upload_chunks`. - let store_one = |addr: [u8; 32], content: Bytes| { + // chunk's merkle proof. Reads the body from the on-disk spill on demand, + // so the whole-file store runs as ONE cap-bounded fan-out with no per-wave + // barrier: a slow straggler (e.g. a chunk whose close-group peers are + // stale relayed addresses that take minutes to revalidate) no longer + // holds back the rest of the file. Only the ≤cap in-flight stores hold a + // body, so peak resident memory is `cap × MAX_CHUNK_SIZE` — the same + // bound the fixed 64-chunk waves gave, since the store cap maxes at 64. + // Shared across every deferred round so a converged routing table yields + // a fresh group. Only a quorum shortfall is recoverable; a missing proof + // or a failed spill read stays fatal. Mirrors `merkle_upload_chunks`. + let store_one = |addr: [u8; 32]| { let limiter = store_limiter.clone(); let proof_bytes = batch_result.proofs.get(&addr).cloned(); async move { @@ -2578,6 +2570,7 @@ impl Client { hex::encode(addr) )) })?; + let content = spill.read_chunk(&addr)?; let peers = self.put_target_peers(&addr).await?; observe_op( &limiter, @@ -2589,135 +2582,93 @@ impl Client { } }; - for (wave_idx, wave_addrs) in waves.into_iter().enumerate() { - let wave_num = wave_idx + 1; - let wave = spill.read_wave(wave_addrs)?; + info!( + "Storing {} chunks (merkle) as a single cap-bounded pass — {total_stored}/{total_chunks} stored so far", + to_store.len() + ); - info!( - "Wave {wave_num}/{wave_count}: storing {} chunks (merkle) — {total_stored}/{total_chunks} stored so far", - wave.len() - ); + // Store the WHOLE file in one cap-bounded fan-out (`max_attempts = 1`, no + // backoff): no wave barrier, so a slow straggler (dead-relay peers) can't + // hold back the rest of the file. The store cap re-reads the limiter per + // slot, so it maxes at 64 → ≤64 bodies resident (bodies read from spill on + // demand by `store_one`), the same peak-memory bound the fixed 64-chunk + // waves gave. Quorum-short chunks are collected and deferred to the + // post-pass concurrent retry rather than parking slots behind a backoff. + let cap = || store_limiter.current().max(1); + let outcome = merkle_store_with_retry( + to_store.clone(), + cap, + 1, + std::time::Duration::ZERO, + progress, + total_stored, + total_chunks, + &store_one, + ) + .await?; + + // Record confirmed stores from the explicit set the store helper reports. + // Using that set (rather than inferring "chunks minus failed") keeps + // `stored_addresses` correct even when a fatal abort leaves some chunks + // neither stored nor reported short of quorum. + stored_addresses.extend(&outcome.stored_addresses); + total_stored = outcome.stored; + + // Merge store stats (durations, attempts, per-round histogram). + agg_stats.chunk_attempts_total = agg_stats + .chunk_attempts_total + .saturating_add(outcome.stats.chunk_attempts_total); + agg_stats + .store_durations_ms + .extend(outcome.stats.store_durations_ms); + for (slot, count) in agg_stats + .retries_histogram + .iter_mut() + .zip(outcome.stats.retries_histogram.iter()) + { + *slot = slot.saturating_add(*count); + } - // Cap closure re-read per scheduler refill so mid-flight limiter - // growth reaches the rest of the wave (V2-554). Clamped to wave size - // — a partial last wave should not pay for extra slots (see - // PERF-RESULTS.md). - let wave_len = wave.len(); - let cap = || store_limiter.current().min(wave_len.max(1)); - let chunks: Vec<([u8; 32], Bytes)> = wave - .into_iter() - .map(|(content, addr)| (addr, content)) - .collect(); - - // Store the wave in a SINGLE pass (`max_attempts = 1`, no backoff): - // quorum-short chunks are collected and deferred to a post-wave - // concurrent retry rather than parking this wave's other slots - // behind a backoff. `stored_offset` is the running cumulative count - // so the progress events the driver emits stay correctly numbered - // across waves. - let outcome = merkle_store_with_retry( - chunks, - cap, - 1, - std::time::Duration::ZERO, - progress, + if let Some(e) = outcome.fatal { + // A non-quorum store error is fatal (missing proofs were filtered out + // above, so this is a genuine network/store failure). Preserve every + // chunk stored so far and report every not-stored chunk as failed, so + // the `PartialUpload` counts are accurate. + warn!("merkle store aborted: {e}"); + let mut known_failed = failed; + known_failed.extend(outcome.failed_addresses); + return Err(partial_upload_after_fatal( + addresses, + stored_addresses, total_stored, total_chunks, - &store_one, - ) - .await?; - - // Record this wave's confirmed stores from the explicit set the - // store helper reports. Using that set (rather than inferring - // "wave chunks minus failed") keeps `stored_addresses` correct even - // when a fatal abort leaves some of the wave neither stored nor - // reported short of quorum. - stored_addresses.extend(&outcome.stored_addresses); - total_stored = outcome.stored; - - // Merge per-wave stats (durations, attempts, per-round histogram). - agg_stats.chunk_attempts_total = agg_stats - .chunk_attempts_total - .saturating_add(outcome.stats.chunk_attempts_total); - agg_stats - .store_durations_ms - .extend(outcome.stats.store_durations_ms); - for (slot, count) in agg_stats - .retries_histogram - .iter_mut() - .zip(outcome.stats.retries_histogram.iter()) - { - *slot = slot.saturating_add(*count); - } - - if let Some(e) = outcome.fatal { - // A non-quorum store error is fatal (missing proofs were - // filtered out above, so this is a genuine network/store - // failure). Preserve every chunk stored so far — including this - // wave's same-pass successes — and report every not-stored chunk - // as failed, so the `PartialUpload` counts are accurate rather - // than omitting same-wave stores and under-counting failures. - warn!("merkle wave {wave_num}/{wave_count} aborted: {e}"); - // Best per-chunk messages we already have: missing-proof, this - // wave's shortfalls, and earlier waves' deferred shortfalls. - let mut known_failed = failed; - known_failed.extend(outcome.failed_addresses); - known_failed.extend(std::mem::take(&mut deferred)); - return Err(partial_upload_after_fatal( - addresses, - stored_addresses, - total_stored, - total_chunks, - known_failed, - PartialUploadSpend { - storage_cost_atto: batch_result.storage_cost_atto.clone(), - gas_cost_wei: batch_result.gas_cost_wei, - }, - format!("merkle chunk store aborted: {e}"), - )); - } - - // Non-fatal: this wave's quorum-short chunks are deferred (not failed - // yet) for the post-wave concurrent retry. A deferred chunk joins - // `stored_addresses` only if/when a later round stores it. - deferred.extend(outcome.failed_addresses); - - if let Some(tx) = progress { - let _ = tx - .send(UploadEvent::WaveComplete { - wave: wave_num, - total_waves: wave_count, - stored_so_far: total_stored, - total: total_chunks, - }) - .await; - } + known_failed, + PartialUploadSpend { + storage_cost_atto: batch_result.storage_cost_atto.clone(), + gas_cost_wei: batch_result.gas_cost_wei, + }, + format!("merkle chunk store aborted: {e}"), + )); } - // The wave passes never blocked on backoff; now retry the whole - // file-level deferred set in concurrent rounds. Bodies are re-read from - // the spill at retry time (peak RAM unchanged) and proofs are re-attached - // by `store_one`. Chunks still short after the final round become - // `failed`; a non-quorum error aborts as `PartialUpload`. + // Non-fatal: quorum-short chunks are deferred (not failed yet) for the + // post-pass concurrent retry. A deferred chunk joins `stored_addresses` + // only if/when a later round stores it. + let deferred: Vec<([u8; 32], String)> = outcome.failed_addresses; + + // The store pass never blocked on backoff; now retry the deferred set in + // concurrent rounds. Bodies are re-read from the spill by `store_one` + // (peak RAM unchanged) and proofs re-attached. Chunks still short after + // the final round become `failed`; a non-quorum error aborts as + // `PartialUpload`. if !deferred.is_empty() { info!( - "Deferring {} merkle chunk(s) short of quorum for concurrent retry after final wave", + "Deferring {} merkle chunk(s) short of quorum for concurrent retry after the store pass", deferred.len() ); let dr = merkle_deferred_retry( deferred, &DEFERRED_ROUND_DELAYS_SECS, - // Read and store at most one wave's worth of bodies at a time so - // the deferred path keeps the wave path's ~256 MB peak-memory - // bound regardless of how many chunks were deferred file-wide. - UPLOAD_WAVE_SIZE, - |addrs: &[[u8; 32]]| { - spill.read_wave(addrs).map(|wave| { - wave.into_iter() - .map(|(content, addr)| (addr, content)) - .collect() - }) - }, |n: usize| store_limiter.current().min(n.max(1)), progress, total_stored, @@ -3571,7 +3522,7 @@ mod tests { } /// A partial merkle payment leaves some addresses without a proof. Those - /// must be split out so `upload_waves_merkle` reports them as failed + /// must be split out so `upload_merkle_from_spill` reports them as failed /// (`PartialUpload`) instead of aborting the whole file — preserving the /// addresses' original order in each group. #[test] diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index dce8cb63..d807128b 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -840,17 +840,27 @@ impl Client { // partial batches should not pay for unused slots (see PERF-RESULTS.md). let cap = || store_limiter.current().min(batch_size.max(1)); - let chunks: Vec<([u8; 32], Bytes)> = addresses.into_iter().zip(chunk_contents).collect(); + // External-signer path: the chunk bodies are already resident in memory, + // so `store_one` clones each from this map on demand. (The whole-file + // path instead reads bodies from the on-disk spill — same `store_one(addr)` + // shape, so the store scheduler never carries a resident set of bodies.) + let bodies: std::collections::HashMap<[u8; 32], Bytes> = + addresses.iter().copied().zip(chunk_contents).collect(); + let addrs = addresses; // Store one chunk to its (freshly re-collected) close group. Called // once per chunk per attempt, so a retry round naturally lands on a // converged routing table. Only `InsufficientPeers` is recoverable; // a missing proof stays fatal. - let store_one = |addr: [u8; 32], content: Bytes| { + let store_one = |addr: [u8; 32]| { let limiter = store_limiter.clone(); + let content = bodies.get(&addr).cloned(); let proof_bytes = batch_result.proofs.get(&addr).cloned(); async move { let started = std::time::Instant::now(); + let content = content.ok_or_else(|| { + Error::InvalidData(format!("missing chunk body for {}", hex::encode(addr))) + })?; let proof = proof_bytes.ok_or_else(|| { Error::Payment(format!( "Missing merkle proof for chunk {}", @@ -869,7 +879,7 @@ impl Client { }; let outcome = merkle_store_with_retry( - chunks, + addrs, cap, MERKLE_STORE_MAX_ATTEMPTS, MERKLE_RETRY_BACKOFF, @@ -950,10 +960,14 @@ pub(crate) struct MerkleStoreOutcome { /// Drive a set of merkle chunk stores with bounded retry of quorum shortfalls. /// -/// Runs `store_one` over all `chunks` concurrently, keeping up to `cap()` stores -/// in flight and RE-READING `cap()` as each slot frees — so mid-flight adaptive -/// growth is applied to the rest of the round instead of being frozen at a -/// per-round snapshot (V2-554). Collects quorum shortfalls +/// Runs `store_one(addr)` over all `addrs` concurrently, keeping up to `cap()` +/// stores in flight and RE-READING `cap()` as each slot frees — so mid-flight +/// adaptive growth is applied to the rest of the round instead of being frozen +/// at a per-round snapshot (V2-554). `store_one` acquires the chunk body itself +/// (e.g. reads it from the on-disk spill), so only the ≤`cap()` in-flight stores +/// hold a body in memory — the caller passes addresses, never a resident set of +/// bodies, which is what lets the whole-file store run as one cap-bounded +/// fan-out instead of memory-bounded waves. Collects quorum shortfalls /// (`InsufficientPeers`, `CloseGroupShortfall`, `RemotePut`) rather than /// aborting. Failed chunks are retried — `store_one` re-collects their close /// group on each call, so a converged routing table can yield a fresh group — @@ -971,7 +985,7 @@ pub(crate) struct MerkleStoreOutcome { /// `Err`; CLI callers fold it into `PartialUpload` while keeping the stores. #[allow(clippy::too_many_arguments)] pub(crate) async fn merkle_store_with_retry( - chunks: Vec<([u8; 32], Bytes)>, + addrs: Vec<[u8; 32]>, cap: C, max_attempts: usize, backoff: Duration, @@ -981,7 +995,7 @@ pub(crate) async fn merkle_store_with_retry( store_one: F, ) -> Result where - F: Fn([u8; 32], Bytes) -> Fut, + F: Fn([u8; 32]) -> Fut, Fut: std::future::Future>, C: Fn() -> usize, { @@ -990,13 +1004,15 @@ where stored: stored_offset, ..MerkleStoreOutcome::default() }; - let mut pending = chunks; + let mut pending = addrs; for attempt in 0..attempts { - // Carries the chunk body forward for the next round plus the last + // Carries the failing address forward for the next round plus the last // quorum-shortfall message, so an exhausted set can report per-chunk - // errors via `failed_addresses`. - let mut next_failed: Vec<([u8; 32], Bytes, String)> = Vec::new(); + // errors via `failed_addresses`. The chunk BODY is not carried — each + // `store_one` re-reads it on demand, so at most `cap` bodies are ever + // resident regardless of the total chunk count. + let mut next_failed: Vec<([u8; 32], String)> = Vec::new(); // Rolling scheduler: keep up to `cap()` stores in flight, re-reading the // cap as each slot frees so limiter growth mid-round is applied to the @@ -1008,14 +1024,14 @@ where let slots = cap().max(1); while in_flight.len() < slots { match pending_iter.next() { - Some((addr, content)) => { - let fut = store_one(addr, content.clone()); - in_flight.push(async move { (addr, content, fut.await) }); + Some(addr) => { + let fut = store_one(addr); + in_flight.push(async move { (addr, fut.await) }); } None => break, } } - let Some((addr, content, result)) = in_flight.next().await else { + let Some((addr, result)) = in_flight.next().await else { break; }; outcome.stats.chunk_attempts_total = @@ -1048,7 +1064,7 @@ where | Error::CloseGroupShortfall(_) | Error::RemotePut { .. }), ) => { - next_failed.push((addr, content, e.to_string())); + next_failed.push((addr, e.to_string())); } Err(e) => { // Non-quorum error: fatal. Stop consuming the stream but do @@ -1058,7 +1074,7 @@ where // far) as failed; anything still in flight is left for the // caller to treat as not-stored (input minus // `stored_addresses`). - next_failed.push((addr, content, e.to_string())); + next_failed.push((addr, e.to_string())); outcome.fatal = Some(e); break; } @@ -1067,10 +1083,7 @@ where if outcome.fatal.is_some() { outcome.failed = next_failed.len(); - outcome.failed_addresses = next_failed - .into_iter() - .map(|(addr, _content, msg)| (addr, msg)) - .collect(); + outcome.failed_addresses = next_failed; return Ok(outcome); } @@ -1084,10 +1097,7 @@ where attempt = attempt + 1, "merkle chunks short of quorum, retrying after backoff" ); - pending = next_failed - .into_iter() - .map(|(addr, content, _msg)| (addr, content)) - .collect(); + pending = next_failed.into_iter().map(|(addr, _msg)| addr).collect(); if backoff > Duration::ZERO { // Jitter the wait (±MERKLE_RETRY_JITTER) so a large failed set // does not re-probe the same divergent nodes in lockstep. @@ -1102,10 +1112,7 @@ where } } else { outcome.failed = next_failed.len(); - outcome.failed_addresses = next_failed - .into_iter() - .map(|(addr, _content, msg)| (addr, msg)) - .collect(); + outcome.failed_addresses = next_failed; break; } } @@ -1161,28 +1168,24 @@ pub(crate) struct DeferredRetryOutcome { /// Retry a file-level set of quorum-short merkle chunks in concurrent rounds. /// /// This is the upload analogue of the download path's deferred-retry loop. The -/// wave passes store each wave in a single pass (no in-wave backoff barrier) and -/// hand their quorum-short chunks here. Each round processes the still-pending -/// chunks in **bounded batches of `batch_size`**: it re-reads only one batch of -/// bodies at a time via `read_bodies` (from the spill file), so peak resident -/// memory stays at the wave path's `batch_size × MAX_CHUNK_SIZE` bound rather -/// than scaling with the whole file's deferred-chunk count. Each batch is stored -/// concurrently at `concurrency_for(len)` via the single-pass -/// [`merkle_store_with_retry`] primitive, and survivors carry to the next round +/// whole-file store pass hands its quorum-short chunks here. Each round stores +/// all still-pending chunks in a single cap-bounded pass via +/// [`merkle_store_with_retry`] at `concurrency_for(len)` — `store_one(addr)` +/// re-reads each body on demand (from the spill), so only the ≤`concurrency_for` +/// in-flight stores hold a body and peak resident memory stays bounded +/// regardless of the deferred-chunk count. Survivors carry to the next round /// after a `round_delays_secs` sleep. Chunks still short after the final round /// become `failed_addresses`; a non-quorum store error stops the pass and is -/// reported via `fatal` (with every not-yet-stored chunk recorded as -/// `failed_addresses`) so the caller can surface `PartialUpload` without -/// discarding earlier progress. +/// reported via `fatal` (with the quorum shortfalls seen so far recorded as +/// `failed_addresses`) so the caller can surface `PartialUpload` — reconciled +/// against the full address list — without discarding earlier progress. /// /// `store_one`, `progress`, `stored_offset` and `total` mirror /// [`merkle_store_with_retry`]. #[allow(clippy::too_many_arguments)] -pub(crate) async fn merkle_deferred_retry( +pub(crate) async fn merkle_deferred_retry( deferred: Vec<([u8; 32], String)>, round_delays_secs: &[u64], - batch_size: usize, - read_bodies: RB, concurrency_for: CF, progress: Option<&mpsc::Sender>, stored_offset: usize, @@ -1190,12 +1193,10 @@ pub(crate) async fn merkle_deferred_retry( store_one: SF, ) -> Result where - RB: Fn(&[[u8; 32]]) -> Result>, CF: Fn(usize) -> usize, - SF: Fn([u8; 32], Bytes) -> Fut, + SF: Fn([u8; 32]) -> Fut, Fut: std::future::Future>, { - let batch_size = batch_size.max(1); let mut outcome = DeferredRetryOutcome { stored: stored_offset, ..DeferredRetryOutcome::default() @@ -1217,75 +1218,62 @@ where remaining.len(), ); - // Drain this round's input; survivors accumulate back into `remaining` - // for the next round. A single-pass batch records its successes in - // histogram slot 0, so all of this round's successes redirect to one - // slot. + // Store this round's whole pending set in one cap-bounded pass. A + // single-pass round records its successes in histogram slot 0, so + // redirect them into the round's own slot. let slot = deferred_round_histogram_slot(round, outcome.stats.retries_histogram.len()); - let round_input = std::mem::take(&mut remaining); - let mut input_iter = round_input.into_iter(); - - loop { - let batch: Vec<([u8; 32], String)> = input_iter.by_ref().take(batch_size).collect(); - if batch.is_empty() { - break; - } - let batch_addrs: Vec<[u8; 32]> = batch.iter().map(|(addr, _)| *addr).collect(); - // Re-read only this batch's bodies from the spill (≤ batch_size - // resident at a time), so the deferred path keeps the wave path's - // memory bound regardless of how many chunks were deferred. - let chunks = read_bodies(&batch_addrs)?; - // Re-read the cap per scheduler refill (V2-554) via `concurrency_for`, - // which re-samples the store limiter clamped to this batch's size. - let cap = || concurrency_for(batch_addrs.len()); - - let batch_outcome = merkle_store_with_retry( - chunks, - cap, - 1, - Duration::ZERO, - progress, - outcome.stored, - total, - &store_one, - ) - .await?; + let round_addrs: Vec<[u8; 32]> = std::mem::take(&mut remaining) + .into_iter() + .map(|(addr, _msg)| addr) + .collect(); + let round_len = round_addrs.len(); + // Re-read the cap per scheduler refill (V2-554) via `concurrency_for`, + // which re-samples the store limiter clamped to this round's size. + let cap = || concurrency_for(round_len); - outcome.stored = batch_outcome.stored; - outcome - .stored_addresses - .extend(batch_outcome.stored_addresses); - - // Merge stats, redirecting this round's successes to its slot. - outcome.stats.chunk_attempts_total = outcome - .stats - .chunk_attempts_total - .saturating_add(batch_outcome.stats.chunk_attempts_total); - outcome - .stats - .store_durations_ms - .extend(batch_outcome.stats.store_durations_ms); - let landed: usize = batch_outcome.stats.retries_histogram.iter().sum(); - outcome.stats.retries_histogram[slot] = - outcome.stats.retries_histogram[slot].saturating_add(landed); - - if let Some(fatal) = batch_outcome.fatal { - // Fatal mid-pass: confirmed stores are preserved above. Report - // everything not stored as failed — this batch's quorum - // shortfalls, the remaining unprocessed batches in this round, - // and any survivors already carried from earlier batches. - outcome.fatal = Some(fatal.to_string()); - let mut failed = batch_outcome.failed_addresses; - failed.extend(input_iter); - failed.extend(std::mem::take(&mut remaining)); - outcome.failed = failed.len(); - outcome.failed_addresses = failed; - return Ok(outcome); - } + let round_outcome = merkle_store_with_retry( + round_addrs, + cap, + 1, + Duration::ZERO, + progress, + outcome.stored, + total, + &store_one, + ) + .await?; - // Quorum-short chunks from this batch survive to the next round. - remaining.extend(batch_outcome.failed_addresses); + outcome.stored = round_outcome.stored; + outcome + .stored_addresses + .extend(round_outcome.stored_addresses); + + // Merge stats, redirecting this round's successes to its slot. + outcome.stats.chunk_attempts_total = outcome + .stats + .chunk_attempts_total + .saturating_add(round_outcome.stats.chunk_attempts_total); + outcome + .stats + .store_durations_ms + .extend(round_outcome.stats.store_durations_ms); + let landed: usize = round_outcome.stats.retries_histogram.iter().sum(); + outcome.stats.retries_histogram[slot] = + outcome.stats.retries_histogram[slot].saturating_add(landed); + + if let Some(fatal) = round_outcome.fatal { + // Fatal mid-pass: confirmed stores are preserved above. The store + // helper left this round's quorum shortfalls in `failed_addresses`; + // chunks still in flight / not yet launched are reconciled against + // the full address list by the caller's `partial_upload_after_fatal`. + outcome.fatal = Some(fatal.to_string()); + outcome.failed = round_outcome.failed_addresses.len(); + outcome.failed_addresses = round_outcome.failed_addresses; + return Ok(outcome); } + + // Quorum-short chunks from this round survive to the next. + remaining = round_outcome.failed_addresses; } outcome.failed = remaining.len(); @@ -1795,12 +1783,10 @@ mod tests { use std::sync::{Arc, Mutex}; - /// Build `count` (addr, content) pairs for the retry helper. - fn make_chunks(count: usize) -> Vec<([u8; 32], Bytes)> { + /// Build `count` chunk addresses for the store helper. Bodies are read on + /// demand by `store_one`, so the tests pass addresses only. + fn make_addrs(count: usize) -> Vec<[u8; 32]> { make_test_addresses(count) - .into_iter() - .map(|addr| (addr, Bytes::from_static(b"chunk"))) - .collect() } /// C2.1: a per-chunk `InsufficientPeers` is collected, not propagated — @@ -1808,12 +1794,11 @@ mod tests { /// subset is reported via `failed` and the rest are `stored`. #[tokio::test] async fn store_with_retry_collects_failures_instead_of_aborting() { - let chunks = make_chunks(6); - let failing: std::collections::HashSet<[u8; 32]> = - chunks.iter().take(2).map(|(a, _)| *a).collect(); + let chunks = make_addrs(6); + let failing: std::collections::HashSet<[u8; 32]> = chunks.iter().take(2).copied().collect(); let failing_for_closure = failing.clone(); - let store_one = move |addr: [u8; 32], _content: Bytes| { + let store_one = move |addr: [u8; 32]| { let fail = failing_for_closure.contains(&addr); async move { if fail { @@ -1844,15 +1829,14 @@ mod tests { #[tokio::test] async fn store_with_retry_rereads_cap_per_slot() { let count = 6; - let chunks = make_chunks(count); + let chunks = make_addrs(count); let cap_calls = Arc::new(Mutex::new(0usize)); let cap_calls_for_closure = cap_calls.clone(); let cap = move || { *cap_calls_for_closure.lock().expect("cap counter poisoned") += 1; 2 }; - let store_one = - move |_addr: [u8; 32], _content: Bytes| async move { Ok(std::time::Instant::now()) }; + let store_one = move |_addr: [u8; 32]| async move { Ok(std::time::Instant::now()) }; let outcome = merkle_store_with_retry(chunks, cap, 1, Duration::ZERO, None, 0, count, store_one) @@ -1868,18 +1852,112 @@ mod tests { ); } + /// The whole-file store pass is a single cap-bounded fan-out with NO wave + /// barrier: one slow straggler (a chunk whose peers take a long time) must + /// not block the rest of the file, and must be driven to completion + /// concurrently by the fast chunks. If the scheduler serialized (a barrier), + /// the fast chunks could not run until the straggler returned — a deadlock + /// the surrounding timeout would catch. + #[tokio::test] + async fn store_pass_has_no_barrier() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let count = 8; + let addrs = make_addrs(count); + let slow = addrs[0]; + let fast_completed = Arc::new(AtomicUsize::new(0)); + let release_slow = Arc::new(tokio::sync::Notify::new()); + + let store_one = move |addr: [u8; 32]| { + let fast_completed = fast_completed.clone(); + let release_slow = release_slow.clone(); + async move { + if addr == slow { + // Block until every fast chunk has finished. This can only + // resolve if the fast chunks run WHILE this one is parked — + // i.e. there is no barrier serializing the store pass. + release_slow.notified().await; + } else if fast_completed.fetch_add(1, Ordering::SeqCst) + 1 == count - 1 { + release_slow.notify_one(); + } + Ok(std::time::Instant::now()) + } + }; + + let outcome = tokio::time::timeout( + Duration::from_secs(5), + merkle_store_with_retry(addrs, || 8, 1, Duration::ZERO, None, 0, count, store_one), + ) + .await + .expect("store pass must not deadlock — a slow chunk must not block the others") + .expect("all stores succeed"); + + assert_eq!(outcome.stored, count); + } + + /// Peak memory is bounded by the store cap, not the file size: `store_one` + /// reads each body on demand, so the scheduler holds at most `cap` stores + /// (hence at most `cap` bodies) in flight at once — the property that lets + /// the whole file store as one fan-out without the old 64-chunk waves. + #[tokio::test] + async fn store_pass_keeps_at_most_cap_in_flight() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let count = 40; + let cap = 4; + let addrs = make_addrs(count); + let in_flight = Arc::new(AtomicUsize::new(0)); + let max_in_flight = Arc::new(AtomicUsize::new(0)); + let max_in_flight_for_closure = max_in_flight.clone(); + + let store_one = move |_addr: [u8; 32]| { + let in_flight = in_flight.clone(); + let max_in_flight = max_in_flight_for_closure.clone(); + async move { + let now = in_flight.fetch_add(1, Ordering::SeqCst) + 1; + max_in_flight.fetch_max(now, Ordering::SeqCst); + // Yield so sibling stores get a chance to start concurrently, + // maximising the observed in-flight count. + tokio::task::yield_now().await; + in_flight.fetch_sub(1, Ordering::SeqCst); + Ok(std::time::Instant::now()) + } + }; + + let outcome = merkle_store_with_retry( + addrs, + move || cap, + 1, + Duration::ZERO, + None, + 0, + count, + store_one, + ) + .await + .expect("all stores succeed"); + + assert_eq!(outcome.stored, count); + let peak = max_in_flight.load(Ordering::SeqCst); + assert!( + peak <= cap, + "at most `cap` bodies may be in flight (memory bound), got peak {peak} > cap {cap}", + ); + assert!( + peak > 1, + "the pass must actually run concurrently, not serialize (peak {peak})", + ); + } + /// V2-468: an app-only quorum shortfall surfaces as `Error::RemotePut` /// (pool-rejected / quote-stale / disk-full — transient), which must be /// treated as recoverable just like `InsufficientPeers`: collected and /// retried, never aborting the whole batch. #[tokio::test] async fn store_with_retry_treats_remote_put_as_recoverable() { - let chunks = make_chunks(6); - let failing: std::collections::HashSet<[u8; 32]> = - chunks.iter().take(2).map(|(a, _)| *a).collect(); + let chunks = make_addrs(6); + let failing: std::collections::HashSet<[u8; 32]> = chunks.iter().take(2).copied().collect(); let failing_for_closure = failing.clone(); - let store_one = move |addr: [u8; 32], _content: Bytes| { + let store_one = move |addr: [u8; 32]| { let fail = failing_for_closure.contains(&addr); async move { if fail { @@ -1909,8 +1987,8 @@ mod tests { /// whether to re-raise it or fold it into `PartialUpload`. #[tokio::test] async fn store_with_retry_reports_non_quorum_errors_as_fatal() { - let chunks = make_chunks(3); - let store_one = |_addr: [u8; 32], _content: Bytes| async move { + let chunks = make_addrs(3); + let store_one = |_addr: [u8; 32]| async move { Err::(Error::Payment("missing proof".into())) }; @@ -1927,9 +2005,9 @@ mod tests { /// aborts fatally. #[tokio::test] async fn store_with_retry_fatal_preserves_same_pass_successes() { - let chunks = make_chunks(6); - let bad = chunks[5].0; - let store_one = move |addr: [u8; 32], _content: Bytes| async move { + let chunks = make_addrs(6); + let bad = chunks[5]; + let store_one = move |addr: [u8; 32]| async move { if addr == bad { Err(Error::Payment("fatal".into())) } else { @@ -1953,17 +2031,16 @@ mod tests { /// C2.2: only the chunks that failed the previous round are retried. #[tokio::test] async fn store_with_retry_retries_only_the_failed_set() { - let chunks = make_chunks(5); + let chunks = make_addrs(5); let total = chunks.len(); - let failing: std::collections::HashSet<[u8; 32]> = - chunks.iter().take(2).map(|(a, _)| *a).collect(); + let failing: std::collections::HashSet<[u8; 32]> = chunks.iter().take(2).copied().collect(); let failing_for_closure = failing.clone(); // Record every (addr) the store op was invoked with, in call order. let calls = Arc::new(Mutex::new(Vec::<[u8; 32]>::new())); let calls_for_closure = calls.clone(); - let store_one = move |addr: [u8; 32], _content: Bytes| { + let store_one = move |addr: [u8; 32]| { let calls = calls_for_closure.clone(); // Fails the first round only; succeeds thereafter. let already_seen = calls.lock().unwrap().iter().filter(|&&a| a == addr).count(); @@ -2000,14 +2077,14 @@ mod tests { /// once as stored and recorded as one retry in `retries_histogram[1]`. #[tokio::test] async fn store_with_retry_counts_retry_success_once_in_histogram() { - let chunks = make_chunks(4); + let chunks = make_addrs(4); let total = chunks.len(); - let flaky_addr = chunks[0].0; + let flaky_addr = chunks[0]; let attempts = Arc::new(Mutex::new(HashMap::<[u8; 32], usize>::new())); let attempts_for_closure = attempts.clone(); - let store_one = move |addr: [u8; 32], _content: Bytes| { + let store_one = move |addr: [u8; 32]| { let attempts = attempts_for_closure.clone(); let n = { let mut m = attempts.lock().unwrap(); @@ -2045,10 +2122,10 @@ mod tests { /// `MERKLE_STORE_MAX_ATTEMPTS` times. #[tokio::test] async fn store_with_retry_reports_all_failed_when_retries_exhausted() { - let chunks = make_chunks(3); + let chunks = make_addrs(3); let total = chunks.len(); - let store_one = |_addr: [u8; 32], _content: Bytes| async move { + let store_one = |_addr: [u8; 32]| async move { Err::(Error::InsufficientPeers("never converges".into())) }; @@ -2078,16 +2155,15 @@ mod tests { /// D (CLI path): when retries are exhausted, `failed_addresses` names /// exactly the still-short-of-quorum chunks (with their last error message) - /// and excludes the ones that stored. This is what `upload_waves_merkle` + /// and excludes the ones that stored. This is what `upload_merkle_from_spill` /// uses to build `PartialUpload`. #[tokio::test] async fn store_with_retry_records_failed_addresses_when_exhausted() { - let chunks = make_chunks(6); - let failing: std::collections::HashSet<[u8; 32]> = - chunks.iter().take(2).map(|(a, _)| *a).collect(); + let chunks = make_addrs(6); + let failing: std::collections::HashSet<[u8; 32]> = chunks.iter().take(2).copied().collect(); let failing_for_closure = failing.clone(); - let store_one = move |addr: [u8; 32], _content: Bytes| { + let store_one = move |addr: [u8; 32]| { let fail = failing_for_closure.contains(&addr); async move { if fail { @@ -2128,10 +2204,9 @@ mod tests { /// `PartialUpload` is raised by the CLI path in that case). #[tokio::test] async fn store_with_retry_failed_addresses_empty_on_full_success() { - let chunks = make_chunks(4); + let chunks = make_addrs(4); let total = chunks.len(); - let store_one = - |_addr: [u8; 32], _content: Bytes| async move { Ok(std::time::Instant::now()) }; + let store_one = |_addr: [u8; 32]| async move { Ok(std::time::Instant::now()) }; let outcome = merkle_store_with_retry( chunks, @@ -2167,16 +2242,6 @@ mod tests { assert_eq!(deferred_round_histogram_slot(9, 4), 3); } - /// Re-read bodies for a deferred set from a fake "spill": every requested - /// address is returned paired with a stub body. Zero delays so tests do not - /// actually sleep between rounds. - fn fake_read_bodies(addrs: &[[u8; 32]]) -> Result> { - Ok(addrs - .iter() - .map(|a| (*a, Bytes::from_static(b"deferred-body"))) - .collect()) - } - fn deferred_set(count: usize) -> Vec<([u8; 32], String)> { make_test_addresses(count) .into_iter() @@ -2194,7 +2259,7 @@ mod tests { // (round 1 → histogram slot 2). let attempts = Arc::new(Mutex::new(HashMap::<[u8; 32], usize>::new())); let attempts_for_closure = attempts.clone(); - let store_one = move |addr: [u8; 32], _content: Bytes| { + let store_one = move |addr: [u8; 32]| { let attempts = attempts_for_closure.clone(); async move { let n = { @@ -2214,8 +2279,6 @@ mod tests { let outcome = merkle_deferred_retry( deferred, &[0, 0, 0], - 64, - fake_read_bodies, |n: usize| n.max(1), None, 0, @@ -2242,15 +2305,13 @@ mod tests { #[tokio::test] async fn deferred_retry_leftovers_become_failed() { let deferred = deferred_set(2); - let store_one = |_addr: [u8; 32], _content: Bytes| async move { + let store_one = |_addr: [u8; 32]| async move { Err::(Error::InsufficientPeers("always short".into())) }; let outcome = merkle_deferred_retry( deferred, &[0, 0, 0], - 64, - fake_read_bodies, |n: usize| n.max(1), None, 0, @@ -2284,7 +2345,7 @@ mod tests { // hits a fatal Payment error on round 1. let attempts = Arc::new(Mutex::new(HashMap::<[u8; 32], usize>::new())); let attempts_for_closure = attempts.clone(); - let store_one = move |addr: [u8; 32], _content: Bytes| { + let store_one = move |addr: [u8; 32]| { let attempts = attempts_for_closure.clone(); async move { let n = { @@ -2306,8 +2367,6 @@ mod tests { let outcome = merkle_deferred_retry( deferred, &[0, 0, 0], - 64, - fake_read_bodies, |n: usize| n.max(1), None, 0, @@ -2328,15 +2387,13 @@ mod tests { /// An empty deferred set is a no-op: no rounds run, nothing stored or failed. #[tokio::test] async fn deferred_retry_empty_set_is_a_noop() { - let store_one = |_addr: [u8; 32], _content: Bytes| async move { + let store_one = |_addr: [u8; 32]| async move { Err::(Error::InsufficientPeers("unused".into())) }; let outcome = merkle_deferred_retry( Vec::new(), &DEFERRED_ROUND_DELAYS_SECS, - 64, - fake_read_bodies, |n: usize| n.max(1), None, 7, @@ -2352,49 +2409,4 @@ mod tests { assert!(outcome.failed_addresses.is_empty()); assert!(outcome.fatal.is_none()); } - - /// The memory-bound guard (V2-466 review finding 1): a deferred set far - /// larger than `batch_size` is read from the spill in batches of at most - /// `batch_size`, so peak resident bodies never scale with the file-wide - /// deferred count. All chunks still store. - #[tokio::test] - async fn deferred_retry_reads_bodies_in_bounded_batches() { - let deferred = deferred_set(10); - let batch_size = 4; - // Record the largest single read_bodies request. - let max_batch = Arc::new(Mutex::new(0usize)); - let max_batch_for_closure = max_batch.clone(); - let read_bodies = move |addrs: &[[u8; 32]]| { - let mut m = max_batch_for_closure.lock().unwrap(); - *m = (*m).max(addrs.len()); - Ok(addrs - .iter() - .map(|a| (*a, Bytes::from_static(b"body"))) - .collect()) - }; - let store_one = - |_addr: [u8; 32], _content: Bytes| async move { Ok(std::time::Instant::now()) }; - - let outcome = merkle_deferred_retry( - deferred, - &[0, 0, 0], - batch_size, - read_bodies, - |n: usize| n.max(1), - None, - 0, - 10, - store_one, - ) - .await - .expect("bounded-batch deferred retry stores everything"); - - assert_eq!(outcome.stored, 10); - assert_eq!(outcome.stored_addresses.len(), 10); - assert_eq!(outcome.failed, 0); - assert!( - *max_batch.lock().unwrap() <= batch_size, - "read_bodies must never be handed more than batch_size addresses at once" - ); - } } From f5a7ebfae7ad63498940660bbeced1a1b595e737 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 2 Jul 2026 15:04:03 +0100 Subject: [PATCH 03/22] fix: bound merkle spill store memory and emit terminal wave-complete event Address PR #137 review (Hermes / @dirvine): - The de-waved `upload_merkle_from_spill` fan-out sized concurrency straight from `store_limiter.current()`. The default store cap is 64 (~256 MB of spilled bodies, matching the old wave bound), but `AdaptiveConfig::sanitize` permits `adaptive.max.store` above 64, so a custom high max could hold many more full chunk bodies resident. Clamp the fan-out (and the deferred-retry rounds) via `merkle_store_cap`, capped at `MERKLE_STORE_MAX_IN_FLIGHT` (64), so peak resident store memory stays at the wave-era ~256 MB regardless of the configured max. Throughput is unaffected at the default cap. - The de-wave removed the per-wave `UploadEvent::WaveComplete` emission, but the event is a public phase-completion signal consumed by the CLI progress adapter and external API clients. Emit a single terminal `WaveComplete` ("wave 1 of 1") after the store pass and deferred retries so consumers keying on it still see the store phase complete. Per-chunk `ChunkStored` events drive the bar as before. New test `merkle_store_cap_clamps_to_memory_bound`. cargo fmt + clippy clean; 374 ant-core lib tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-core/src/data/client/file.rs | 65 ++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index 1bf94b07..8066f16b 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -170,6 +170,22 @@ struct FileDownloadFetchContext { /// Number of chunks per upload wave (matches batch.rs PAYMENT_WAVE_SIZE). const UPLOAD_WAVE_SIZE: usize = 64; +/// Hard ceiling on chunk bodies held in memory at once by the merkle whole-file +/// store fan-out (`upload_merkle_from_spill`). Each in-flight store holds one +/// spilled body (≤ `MAX_CHUNK_SIZE` = 4 MiB), so this bounds peak resident store +/// memory at ~256 MiB — the same bound the old fixed 64-chunk waves gave. The +/// adaptive store cap can legitimately exceed this (`AdaptiveConfig::sanitize` +/// permits `adaptive.max.store` above 64), so the fan-out clamps its cap here to +/// keep a high configured max from pinning gigabytes of chunk bodies (PR #137 +/// review). Throughput is unaffected at the default cap, which is already 64. +const MERKLE_STORE_MAX_IN_FLIGHT: usize = 64; + +/// The merkle whole-file store fan-out concurrency: the adaptive store cap, +/// clamped to [`MERKLE_STORE_MAX_IN_FLIGHT`] (memory bound) and floored at 1. +fn merkle_store_cap(limiter_current: usize) -> usize { + limiter_current.clamp(1, MERKLE_STORE_MAX_IN_FLIGHT) +} + /// Stream decrypt batches should be larger than fetch fan-out so /// the rolling fetch scheduler can keep launching new chunk GETs as earlier /// ones complete, instead of stopping at each self-encryption batch boundary. @@ -2554,8 +2570,10 @@ impl Client { // barrier: a slow straggler (e.g. a chunk whose close-group peers are // stale relayed addresses that take minutes to revalidate) no longer // holds back the rest of the file. Only the ≤cap in-flight stores hold a - // body, so peak resident memory is `cap × MAX_CHUNK_SIZE` — the same - // bound the fixed 64-chunk waves gave, since the store cap maxes at 64. + // body, so peak resident memory is `cap × MAX_CHUNK_SIZE`; the cap is + // clamped to `MERKLE_STORE_MAX_IN_FLIGHT` (below) so it stays within the + // ~256 MiB bound the fixed 64-chunk waves gave even if `adaptive.max.store` + // is configured above 64. // Shared across every deferred round so a converged routing table yields // a fresh group. Only a quorum shortfall is recoverable; a missing proof // or a failed spill read stays fatal. Mirrors `merkle_upload_chunks`. @@ -2594,7 +2612,10 @@ impl Client { // demand by `store_one`), the same peak-memory bound the fixed 64-chunk // waves gave. Quorum-short chunks are collected and deferred to the // post-pass concurrent retry rather than parking slots behind a backoff. - let cap = || store_limiter.current().max(1); + // `merkle_store_cap` clamps to `MERKLE_STORE_MAX_IN_FLIGHT` so a high + // configured `adaptive.max.store` can't hold more than the wave-era + // ~256 MB of spilled bodies resident (PR #137 review). + let cap = || merkle_store_cap(store_limiter.current()); let outcome = merkle_store_with_retry( to_store.clone(), cap, @@ -2669,7 +2690,7 @@ impl Client { let dr = merkle_deferred_retry( deferred, &DEFERRED_ROUND_DELAYS_SECS, - |n: usize| store_limiter.current().min(n.max(1)), + |n: usize| merkle_store_cap(store_limiter.current()).min(n.max(1)), progress, total_stored, total_chunks, @@ -2719,10 +2740,27 @@ impl Client { failed.extend(dr.failed_addresses); } + // Emit a single terminal `WaveComplete`. The whole-file store now runs + // as one cap-bounded pass rather than fixed waves, but `WaveComplete` is + // a public phase-completion signal (consumed by the CLI progress adapter + // and external API clients), so report it once — "wave 1 of 1" — after + // the store pass and deferred retries so consumers that key on it still + // see the store phase complete (PR #137 review). Per-chunk `ChunkStored` + // events drive the progress bar as before. + if let Some(tx) = progress { + let _ = tx + .send(UploadEvent::WaveComplete { + wave: 1, + total_waves: 1, + stored_so_far: total_stored, + total: total_chunks, + }) + .await; + } + // A file with any permanently-failed chunk is not fully stored — surface - // it as `PartialUpload`, but only after the single wave pass and every - // deferred retry round are exhausted (never silently succeed with - // missing chunks). + // it as `PartialUpload`, but only after the store pass and every deferred + // retry round are exhausted (never silently succeed with missing chunks). if !failed.is_empty() { let failed_count = failed.len(); let total_attempts = 1 + DEFERRED_ROUND_DELAYS_SECS.len(); @@ -3407,6 +3445,19 @@ impl Client { mod tests { use super::*; + #[test] + fn merkle_store_cap_clamps_to_memory_bound() { + // Below the ceiling: pass the adaptive cap through unchanged. + assert_eq!(merkle_store_cap(8), 8); + assert_eq!(merkle_store_cap(64), 64); + // A configured `adaptive.max.store` above the ceiling must be clamped so + // the whole-file fan-out can't pin more than ~256 MB of bodies (PR #137). + assert_eq!(merkle_store_cap(512), MERKLE_STORE_MAX_IN_FLIGHT); + assert_eq!(merkle_store_cap(usize::MAX), MERKLE_STORE_MAX_IN_FLIGHT); + // Never zero — always make progress. + assert_eq!(merkle_store_cap(0), 1); + } + #[test] fn distributed_sample_indices_spreads_across_large_file() { // cap 5 over 100 chunks: first and last included, evenly spread. From feec056de9b54cbb10ea524cdf1c74ece1859cec Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 2 Jul 2026 16:29:04 +0100 Subject: [PATCH 04/22] refactor!: remove the unused UploadEvent::WaveComplete event Follow-up to the PR #137 review discussion. `WaveComplete` had no real consumer: the CLI handler only mirrored `ChunkStored` (`set_position(stored)`), and the ant-ui Tauri backend maps it to `None` (never forwarded to the frontend). Emitting a synthetic "wave 1 of 1" from the de-waved merkle path to preserve it was therefore pointless. Remove the event entirely instead: drop the `UploadEvent::WaveComplete` variant, both emissions (the single/wave-batch path and the merkle terminal one added in the previous commit), and the redundant CLI match arm. Per-chunk `ChunkStored` drives all upload progress bars, so behaviour is unchanged. `UploadEvent` is an internal in-process type (derives only Debug/Clone, not serialised), so this is not an API-schema change; ant-ui pins ant-core by tag and will drop its own dead match arm when it next bumps. cargo fmt + clippy clean; 374 ant-core lib tests pass. BREAKING CHANGE: The `UploadEvent::WaveComplete` variant is removed from ant-core's public `UploadEvent` enum. Any consumer that matches on it must drop that arm; upload progress is fully covered by `UploadEvent::ChunkStored`. Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-cli/src/commands/data/file.rs | 7 ------ ant-core/src/data/client/file.rs | 36 ------------------------------- 2 files changed, 43 deletions(-) diff --git a/ant-cli/src/commands/data/file.rs b/ant-cli/src/commands/data/file.rs index 6d949a7a..1da346dc 100644 --- a/ant-cli/src/commands/data/file.rs +++ b/ant-cli/src/commands/data/file.rs @@ -426,13 +426,6 @@ async fn drive_upload_progress( let pos = std::cmp::max(pb.position(), stored as u64); pb.set_position(pos); } - UploadEvent::WaveComplete { - stored_so_far, - total: _, - .. - } => { - pb.set_position(stored_so_far as u64); - } } } diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index 8066f16b..c5c28ccf 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -61,13 +61,6 @@ pub enum UploadEvent { ChunkQuoted { quoted: usize, total: usize }, /// A chunk has been stored on the network. ChunkStored { stored: usize, total: usize }, - /// A wave has completed. - WaveComplete { - wave: usize, - total_waves: usize, - stored_so_far: usize, - total: usize, - }, } /// Progress events emitted during file download for UI feedback. @@ -2455,17 +2448,6 @@ impl Client { { *slot = slot.saturating_add(*count); } - - if let Some(tx) = progress { - let _ = tx - .send(UploadEvent::WaveComplete { - wave: wave_num, - total_waves: wave_count, - stored_so_far: total_stored, - total: total_chunks, - }) - .await; - } } // Any chunk still failed after every wave was attempted means the file @@ -2740,24 +2722,6 @@ impl Client { failed.extend(dr.failed_addresses); } - // Emit a single terminal `WaveComplete`. The whole-file store now runs - // as one cap-bounded pass rather than fixed waves, but `WaveComplete` is - // a public phase-completion signal (consumed by the CLI progress adapter - // and external API clients), so report it once — "wave 1 of 1" — after - // the store pass and deferred retries so consumers that key on it still - // see the store phase complete (PR #137 review). Per-chunk `ChunkStored` - // events drive the progress bar as before. - if let Some(tx) = progress { - let _ = tx - .send(UploadEvent::WaveComplete { - wave: 1, - total_waves: 1, - stored_so_far: total_stored, - total: total_chunks, - }) - .await; - } - // A file with any permanently-failed chunk is not fully stored — surface // it as `PartialUpload`, but only after the store pass and every deferred // retry round are exhausted (never silently succeed with missing chunks). From fb9909565a574d513d64320e61e043a63c244352 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 30 Jun 2026 01:15:36 +0100 Subject: [PATCH 05/22] feat: evict a node when its disk runs low Production nodes are running out of disk space and rejecting chunk uploads. When free space at a node's data directory falls to 500MB (the same reserve at which ant-node itself refuses to store), the daemon now automatically stops the smallest co-located node and deletes its data directory to reclaim space for the others. The smallest node is chosen to minimise the re-replication cascade on the network. A node that is the only one on its partition is never evicted; that surfaces as critical fleet health instead, so an operator can act manually. Evicted nodes get a persisted `evicted` status with an explanatory reason that survives daemon restarts, are never restarted, and can be dismissed from the list. An extensible fleet-health layer (green/warning/critical) surfaces disk pressure ahead of time and names the next eviction candidate; it is shown in `ant node status` and served at GET /api/v1/health. - Add NodeStatus::Evicted + persisted EvictionRecord on NodeConfig (serde-default, backwards compatible) - Add daemon/disk.rs: per-partition measurement + smallest-node candidate selection (shared by the monitor and the health layer so they never disagree) - Add daemon/health.rs: extensible FleetHealth model, disk check first, fixed 500MB/1024MB internal constants (deliberately not user-configurable) - Add the eviction monitor (stop -> delete data dir -> persist marker -> emit event), NodeEvicted and FleetHealthChanged events, and GET /api/v1/health - Add `ant node dismiss ` and a fleet-health summary line in `ant node status` Tested: cargo test -p ant-core is green, including the new disk/health/eviction/types unit tests and the daemon integration test. The only workspace failure was an unrelated OOM/SIGKILL on the gigabyte-scale e2e_huge_file upload test (client data path, untouched by this change). Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-cli/src/commands/node/dismiss.rs | 39 +++ ant-cli/src/commands/node/mod.rs | 4 + ant-cli/src/commands/node/status.rs | 47 +++- ant-cli/src/main.rs | 3 + ant-core/src/error.rs | 3 + ant-core/src/node/daemon/client.rs | 46 +++- ant-core/src/node/daemon/disk.rs | 257 ++++++++++++++++++++ ant-core/src/node/daemon/health.rs | 317 +++++++++++++++++++++++++ ant-core/src/node/daemon/mod.rs | 2 + ant-core/src/node/daemon/server.rs | 74 +++++- ant-core/src/node/daemon/supervisor.rs | 232 +++++++++++++++++- ant-core/src/node/events.rs | 17 ++ ant-core/src/node/mod.rs | 73 +++++- ant-core/src/node/registry.rs | 1 + ant-core/src/node/types.rs | 90 +++++++ 15 files changed, 1179 insertions(+), 26 deletions(-) create mode 100644 ant-cli/src/commands/node/dismiss.rs create mode 100644 ant-core/src/node/daemon/disk.rs create mode 100644 ant-core/src/node/daemon/health.rs diff --git a/ant-cli/src/commands/node/dismiss.rs b/ant-cli/src/commands/node/dismiss.rs new file mode 100644 index 00000000..ec5e3314 --- /dev/null +++ b/ant-cli/src/commands/node/dismiss.rs @@ -0,0 +1,39 @@ +use clap::Args; +use colored::Colorize; + +use ant_core::node::daemon::client; +use ant_core::node::types::DaemonConfig; + +#[derive(Args)] +pub struct DismissArgs { + /// The ID of the evicted node to dismiss (remove it from the registry/list). + pub node_id: u32, +} + +impl DismissArgs { + pub async fn execute(self, json_output: bool) -> anyhow::Result<()> { + let config = DaemonConfig::default(); + + // Dual-path: go through the daemon when it's running so its in-memory registry stays in + // sync; otherwise operate directly on the registry file. + let status = client::status(&config).await?; + let result = if status.running { + client::dismiss_node(&config, self.node_id).await? + } else { + ant_core::node::remove_node(self.node_id, &config.registry_path)? + }; + + if json_output { + println!("{}", serde_json::to_string_pretty(&result)?); + } else { + println!( + "{} Dismissed node {} ({})", + "✓".green().bold(), + result.removed.id.to_string().bold(), + result.removed.service_name.dimmed() + ); + } + + Ok(()) + } +} diff --git a/ant-cli/src/commands/node/mod.rs b/ant-cli/src/commands/node/mod.rs index c30342be..0273c85b 100644 --- a/ant-cli/src/commands/node/mod.rs +++ b/ant-cli/src/commands/node/mod.rs @@ -1,5 +1,6 @@ pub mod add; pub mod daemon; +pub mod dismiss; pub mod reset; pub mod start; pub mod status; @@ -9,6 +10,7 @@ use clap::Subcommand; use crate::commands::node::add::AddArgs; use crate::commands::node::daemon::DaemonCommand; +use crate::commands::node::dismiss::DismissArgs; use crate::commands::node::reset::ResetArgs; use crate::commands::node::start::StartArgs; use crate::commands::node::status::StatusArgs; @@ -23,6 +25,8 @@ pub enum NodeCommand { #[command(subcommand)] command: DaemonCommand, }, + /// Dismiss an evicted node, removing it from the registry/list + Dismiss(DismissArgs), /// Reset all node state (removes all data, logs, and clears the registry) Reset(ResetArgs), /// Start node(s). With no arguments starts all nodes; use --service-name for a specific node. diff --git a/ant-cli/src/commands/node/status.rs b/ant-cli/src/commands/node/status.rs index 877a7186..8b0e5a56 100644 --- a/ant-cli/src/commands/node/status.rs +++ b/ant-cli/src/commands/node/status.rs @@ -2,6 +2,7 @@ use clap::Args; use colored::Colorize; use ant_core::node::daemon::client; +use ant_core::node::daemon::health::{FleetHealth, HealthLevel}; use ant_core::node::types::{DaemonConfig, NodeStatus}; #[derive(Args)] @@ -18,9 +19,27 @@ impl StatusArgs { ant_core::node::node_status_offline(&config.registry_path)? }; + // Fleet health is only meaningful while the daemon is running (it owns the disk monitor). + let health = if daemon_status.running { + client::fleet_health(&config).await.ok() + } else { + None + }; + if json_output { - println!("{}", serde_json::to_string_pretty(&result)?); + // Fold health into the JSON payload alongside node status. + let payload = serde_json::json!({ + "nodes": result.nodes, + "total_running": result.total_running, + "total_stopped": result.total_stopped, + "health": health, + }); + println!("{}", serde_json::to_string_pretty(&payload)?); + return Ok(()); } else { + if let Some(health) = &health { + print_fleet_health(health); + } if result.nodes.is_empty() { println!( "{} No nodes registered. Add nodes first with: {}", @@ -50,6 +69,7 @@ impl StatusArgs { NodeStatus::UpgradeScheduled => { format!("{} {}", "●".cyan(), "Upgrade scheduled".cyan()) } + NodeStatus::Evicted => format!("{} {}", "●".magenta(), "Evicted".magenta()), }; let version_display = match &node.pending_version { Some(pending) => format!("{} → {}", node.version, pending), @@ -62,6 +82,15 @@ impl StatusArgs { version_display.dimmed(), status_display ); + // Supplementary text explaining an eviction, plus how to clear it. + if let Some(eviction) = &node.eviction { + println!(" {}", eviction.reason.dimmed()); + println!( + " {} {}", + "dismiss with:".dimmed(), + format!("ant node dismiss {}", node.node_id).cyan() + ); + } } if !daemon_status.running { @@ -77,3 +106,19 @@ impl StatusArgs { Ok(()) } } + +/// Print the fleet health summary: an overall line, plus a detail line per non-green check. +fn print_fleet_health(health: &FleetHealth) { + let (dot, label) = match health.overall { + HealthLevel::Green => ("●".green(), "Healthy".green()), + HealthLevel::Warning => ("●".yellow(), "Warning".yellow()), + HealthLevel::Critical => ("●".red(), "Critical".red()), + }; + println!(" {} Fleet health: {}", dot, label); + + // Surface the reason(s) whenever the fleet is not fully healthy. + for check in health.checks.iter().filter(|c| c.level != HealthLevel::Green) { + println!(" {} {}", "→".dimmed(), check.summary.dimmed()); + } + println!(); +} diff --git a/ant-cli/src/main.rs b/ant-cli/src/main.rs index a4bf5dd9..b8523a3b 100644 --- a/ant-cli/src/main.rs +++ b/ant-cli/src/main.rs @@ -110,6 +110,9 @@ async fn run() -> anyhow::Result<()> { commands::node::NodeCommand::Daemon { command } => { command.execute(json).await?; } + commands::node::NodeCommand::Dismiss(args) => { + args.execute(json).await?; + } commands::node::NodeCommand::Reset(args) => { args.execute(json).await?; } diff --git a/ant-core/src/error.rs b/ant-core/src/error.rs index b4031676..2b62460f 100644 --- a/ant-core/src/error.rs +++ b/ant-core/src/error.rs @@ -17,6 +17,9 @@ pub enum Error { #[error("Node not running: {0}")] NodeNotRunning(u32), + #[error("Node {0} has been evicted; its data directory was deleted to reclaim disk space. Dismiss it and add a new node instead of restarting.")] + NodeEvicted(u32), + #[error("Daemon already running (pid: {0})")] DaemonAlreadyRunning(u32), diff --git a/ant-core/src/node/daemon/client.rs b/ant-core/src/node/daemon/client.rs index 2331d572..eeb81a4f 100644 --- a/ant-core/src/node/daemon/client.rs +++ b/ant-core/src/node/daemon/client.rs @@ -2,10 +2,11 @@ use std::path::Path; use std::time::Duration; use crate::error::{Error, Result}; +use crate::node::daemon::health::FleetHealth; use crate::node::process::detach; use crate::node::types::{ DaemonConfig, DaemonInfo, DaemonStartResult, DaemonStatus, DaemonStopResult, NodeStarted, - NodeStatusResult, NodeStopped, StartNodeResult, StopNodeResult, + NodeStatusResult, NodeStopped, RemoveNodeResult, StartNodeResult, StopNodeResult, }; /// Get the daemon's current status by querying its REST API. @@ -204,6 +205,49 @@ pub async fn stop_node(config: &DaemonConfig, node_id: u32) -> Result Result { + let port = read_port_file(&config.port_file_path).ok_or(Error::DaemonNotRunning)?; + + let url = format!("http://127.0.0.1:{port}/api/v1/nodes/{node_id}"); + let resp = reqwest::Client::new() + .delete(&url) + .send() + .await + .map_err(|e| Error::HttpRequest(e.to_string()))?; + + if resp.status().is_success() { + resp.json::() + .await + .map_err(|e| Error::HttpRequest(e.to_string())) + } else { + let body = resp.text().await.unwrap_or_default(); + Err(Error::HttpRequest(body)) + } +} + +/// Get the current fleet health snapshot via the daemon REST API. +pub async fn fleet_health(config: &DaemonConfig) -> Result { + let port = read_port_file(&config.port_file_path).ok_or(Error::DaemonNotRunning)?; + + let url = format!("http://127.0.0.1:{port}/api/v1/health"); + let resp = reqwest::get(&url) + .await + .map_err(|e| Error::HttpRequest(e.to_string()))?; + + if resp.status().is_success() { + resp.json::() + .await + .map_err(|e| Error::HttpRequest(e.to_string())) + } else { + let body = resp.text().await.unwrap_or_default(); + Err(Error::HttpRequest(body)) + } +} + /// Get the status of all registered nodes via the daemon REST API. pub async fn node_status(config: &DaemonConfig) -> Result { let port = read_port_file(&config.port_file_path).ok_or(Error::DaemonNotRunning)?; diff --git a/ant-core/src/node/daemon/disk.rs b/ant-core/src/node/daemon/disk.rs new file mode 100644 index 00000000..eb6321f5 --- /dev/null +++ b/ant-core/src/node/daemon/disk.rs @@ -0,0 +1,257 @@ +//! Disk-space measurement and eviction-candidate selection. +//! +//! This module is the single source of truth for two questions: +//! 1. How much free space is left on each partition that hosts node data? +//! 2. Which node should be evicted next to reclaim space? +//! +//! Both the low-disk eviction monitor (which actually evicts) and the fleet health layer (which +//! warns the user *before* an eviction and names the candidate) call into here, so they can never +//! disagree about who is next. +//! +//! Nodes that live on the same filesystem partition share its free space, so eviction is always +//! decided per partition: evicting one node frees space for every other node on that partition. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +/// Measured disk usage of a single node's data directory. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NodeDiskUsage { + pub node_id: u32, + pub data_dir: PathBuf, + /// Total bytes occupied by the node's data directory on disk (≈ space reclaimable by eviction). + pub size_bytes: u64, +} + +/// Opaque identifier for the filesystem partition a path lives on. +/// +/// On Unix this is the device id of the path (or its nearest existing ancestor); elsewhere it falls +/// back to the canonicalized path prefix. Two paths with the same key share free space. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct PartitionKey(String); + +impl std::fmt::Display for PartitionKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl PartitionKey { + /// Construct a key with an arbitrary label. Test-only — real keys come from [`partition_key`]. + #[cfg(test)] + pub(crate) fn for_test(label: &str) -> Self { + PartitionKey(label.to_string()) + } +} + +/// State of one partition that hosts node data: how much space is free and which running nodes live +/// on it (with their measured sizes). +#[derive(Debug, Clone)] +pub struct PartitionState { + pub partition: PartitionKey, + /// Free bytes available on the partition. + pub available_bytes: u64, + /// Running nodes whose data directory lives on this partition. + pub nodes: Vec, +} + +impl PartitionState { + /// The node that should be evicted first to reclaim space on this partition. + /// + /// Selection rule: the **smallest** data directory wins, because evicting a node forces the + /// network to re-replicate everything it held — so we minimise that cascade by dropping the + /// node that holds the least. Ties are broken by preferring the **newest** node (highest id, + /// least established in the network). Returns `None` if the partition hosts no nodes. + pub fn eviction_candidate(&self) -> Option<&NodeDiskUsage> { + self.nodes.iter().min_by(|a, b| { + a.size_bytes + .cmp(&b.size_bytes) + .then_with(|| b.node_id.cmp(&a.node_id)) + }) + } +} + +/// Recursively sum the size of every regular file under `path`. +/// +/// Symlinks are not followed. Returns 0 if `path` does not exist. Best-effort: entries that cannot +/// be read are skipped rather than aborting the walk, so a transient permission error on one file +/// does not corrupt the eviction decision. +pub fn dir_size(path: &Path) -> u64 { + let mut total = 0u64; + let mut stack = vec![path.to_path_buf()]; + while let Some(dir) = stack.pop() { + let entries = match std::fs::read_dir(&dir) { + Ok(entries) => entries, + Err(_) => continue, + }; + for entry in entries.flatten() { + // `symlink_metadata` does not traverse symlinks, so we never double-count or escape the + // tree via a link. + let meta = match entry.metadata() { + Ok(meta) => meta, + Err(_) => continue, + }; + if meta.is_dir() { + stack.push(entry.path()); + } else if meta.is_file() { + total = total.saturating_add(meta.len()); + } + } + } + total +} + +/// Free bytes on the partition hosting `path` (or its nearest existing ancestor). +/// +/// Returns `None` if the available space cannot be determined. +pub fn available_space(path: &Path) -> Option { + let existing = nearest_existing_ancestor(path); + fs2::available_space(&existing).ok() +} + +/// Group the given running nodes by partition, measuring each node's data-directory size and each +/// partition's free space. +/// +/// `nodes` should be the set of currently-running nodes as `(id, data_dir)` pairs; evicted or +/// stopped nodes are irrelevant to a reclaim decision and should be filtered out by the caller. +/// Partitions whose free space cannot be read are skipped. +pub fn partition_states(nodes: I) -> Vec +where + I: IntoIterator, +{ + let mut grouped: BTreeMap> = BTreeMap::new(); + for (node_id, data_dir) in nodes { + let key = partition_key(&data_dir); + let size_bytes = dir_size(&data_dir); + grouped.entry(key).or_default().push(NodeDiskUsage { + node_id, + data_dir, + size_bytes, + }); + } + + grouped + .into_iter() + .filter_map(|(partition, nodes)| { + // Every node in the group shares the partition, so any node's data_dir answers the + // free-space query. + let probe = nodes.first()?.data_dir.clone(); + let available_bytes = available_space(&probe)?; + Some(PartitionState { + partition, + available_bytes, + nodes, + }) + }) + .collect() +} + +/// Identify the partition a path lives on. +fn partition_key(path: &Path) -> PartitionKey { + let existing = nearest_existing_ancestor(path); + + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if let Ok(meta) = std::fs::metadata(&existing) { + return PartitionKey(format!("dev:{}", meta.dev())); + } + } + + // Fallback (and the non-Unix path): the canonicalized prefix is a reasonable partition proxy. + let canon = std::fs::canonicalize(&existing).unwrap_or(existing); + let key = canon + .components() + .next() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .unwrap_or_else(|| canon.to_string_lossy().into_owned()); + PartitionKey(key) +} + +/// Walk up `path` until an existing directory/file is found, since a node's `data_dir` may not yet +/// exist (or may have just been deleted). Falls back to the path itself if nothing exists. +fn nearest_existing_ancestor(path: &Path) -> PathBuf { + let mut current = Some(path); + while let Some(p) = current { + if p.exists() { + return p.to_path_buf(); + } + current = p.parent(); + } + path.to_path_buf() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn usage(node_id: u32, size_bytes: u64) -> NodeDiskUsage { + NodeDiskUsage { + node_id, + data_dir: PathBuf::from(format!("/data/node-{node_id}")), + size_bytes, + } + } + + fn partition_with(nodes: Vec) -> PartitionState { + PartitionState { + partition: PartitionKey("test".to_string()), + available_bytes: 0, + nodes, + } + } + + #[test] + fn candidate_picks_smallest_data_dir() { + let p = partition_with(vec![usage(1, 900), usage(2, 100), usage(3, 500)]); + assert_eq!(p.eviction_candidate().unwrap().node_id, 2); + } + + #[test] + fn candidate_tie_break_prefers_newest_id() { + // Nodes 1 and 4 are tied on size; the newer (id 4) should be chosen. + let p = partition_with(vec![usage(1, 100), usage(4, 100), usage(2, 800)]); + assert_eq!(p.eviction_candidate().unwrap().node_id, 4); + } + + #[test] + fn candidate_none_when_empty() { + let p = partition_with(vec![]); + assert!(p.eviction_candidate().is_none()); + } + + #[test] + fn dir_size_sums_nested_files() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("a.bin"), vec![0u8; 1000]).unwrap(); + let sub = tmp.path().join("sub"); + std::fs::create_dir(&sub).unwrap(); + std::fs::write(sub.join("b.bin"), vec![0u8; 2500]).unwrap(); + assert_eq!(dir_size(tmp.path()), 3500); + } + + #[test] + fn dir_size_zero_for_missing_path() { + assert_eq!(dir_size(Path::new("/nonexistent/path/xyz")), 0); + } + + #[test] + fn partition_states_groups_colocated_nodes() { + // Two nodes under the same temp dir share a partition. + let tmp = tempfile::tempdir().unwrap(); + let d1 = tmp.path().join("node-1"); + let d2 = tmp.path().join("node-2"); + std::fs::create_dir_all(&d1).unwrap(); + std::fs::create_dir_all(&d2).unwrap(); + std::fs::write(d1.join("data"), vec![0u8; 4000]).unwrap(); + std::fs::write(d2.join("data"), vec![0u8; 1000]).unwrap(); + + let states = partition_states(vec![(1, d1), (2, d2)]); + assert_eq!(states.len(), 1, "co-located nodes share one partition"); + let state = &states[0]; + assert_eq!(state.nodes.len(), 2); + assert!(state.available_bytes > 0); + // Node 2 is smaller, so it is the candidate. + assert_eq!(state.eviction_candidate().unwrap().node_id, 2); + } +} diff --git a/ant-core/src/node/daemon/health.rs b/ant-core/src/node/daemon/health.rs new file mode 100644 index 00000000..fc7f8c61 --- /dev/null +++ b/ant-core/src/node/daemon/health.rs @@ -0,0 +1,317 @@ +//! Fleet health: an extensible, always-available indicator of node-fleet health. +//! +//! The model is deliberately generic — [`FleetHealth`] is a list of [`HealthCheck`]s plus an +//! `overall` level — so future signals (connectivity, sync lag, reward health, …) can be added as +//! new [`HealthCheckKind`]s without changing the API surface the CLI and GUI consume. +//! +//! The first and only check today is **disk space**. It answers, per partition: are we comfortable +//! (green), is an eviction likely soon (warning, with the candidate named), or is the partition at +//! the eviction threshold (critical)? The candidate it names is computed by [`super::disk`] — the +//! exact same selection the eviction monitor uses, so the warning never points at a different node +//! than the one that actually gets evicted. + +use serde::{Deserialize, Serialize}; + +use super::disk::PartitionState; + +/// One mebibyte, in bytes. +pub const MIB: u64 = 1024 * 1024; + +/// Fixed free-space floor at which the daemon evicts a node, mirroring the node's own refuse-to-store +/// reserve in `ant-node`'s storage layer. Internal constant — deliberately not user-configurable. +const EVICTION_THRESHOLD_MB: u64 = 500; + +/// Fixed free-space level at which the fleet health turns to `Warning` and names the node that would +/// be evicted next. Internal constant — deliberately not user-configurable. +const WARNING_THRESHOLD_MB: u64 = 1024; + +/// Severity level for a single check or the fleet as a whole. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum HealthLevel { + /// Everything is comfortable. + Green, + /// Action may be needed soon (e.g. an eviction is approaching). + Warning, + /// At or past a hard limit right now (e.g. eviction is imminent, or space is exhausted and the + /// daemon cannot help automatically). + Critical, +} + +impl HealthLevel { + fn rank(self) -> u8 { + match self { + HealthLevel::Green => 0, + HealthLevel::Warning => 1, + HealthLevel::Critical => 2, + } + } + + /// The more severe of two levels. + pub fn worst(self, other: HealthLevel) -> HealthLevel { + if other.rank() > self.rank() { + other + } else { + self + } + } +} + +/// Which signal produced a [`HealthCheck`]. Extensible: new variants slot in here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum HealthCheckKind { + /// Free disk space at node data directories. + DiskSpace, +} + +/// The node a check has identified as the next eviction candidate. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct EvictionCandidate { + pub node_id: u32, + #[schema(value_type = String)] + pub data_dir: String, + /// Bytes the candidate's data directory currently occupies (≈ space its eviction would free). + pub size_bytes: u64, +} + +/// A single health finding. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct HealthCheck { + pub kind: HealthCheckKind, + pub level: HealthLevel, + /// Human-readable, user-facing one-liner. + pub summary: String, + /// Partition this finding concerns (disk checks only). + #[serde(skip_serializing_if = "Option::is_none")] + pub partition: Option, + /// Free bytes on the partition (disk checks only). + #[serde(skip_serializing_if = "Option::is_none")] + pub available_bytes: Option, + /// Free-space floor at which an eviction triggers (disk checks only). + #[serde(skip_serializing_if = "Option::is_none")] + pub eviction_threshold_bytes: Option, + /// The node that would be evicted next, when one applies. + #[serde(skip_serializing_if = "Option::is_none")] + pub candidate: Option, +} + +/// Fleet-wide health snapshot: the worst level across all checks, plus the individual findings. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct FleetHealth { + pub overall: HealthLevel, + pub checks: Vec, +} + +impl FleetHealth { + /// A green snapshot with no findings (e.g. no nodes registered). + pub fn healthy() -> Self { + FleetHealth { + overall: HealthLevel::Green, + checks: Vec::new(), + } + } + + /// Build the fleet health from measured partition states and the configured disk thresholds. + pub fn from_partitions(partitions: &[PartitionState], thresholds: &DiskThresholds) -> Self { + let checks: Vec = partitions + .iter() + .map(|p| disk_space_check(p, thresholds)) + .collect(); + let overall = checks + .iter() + .fold(HealthLevel::Green, |acc, c| acc.worst(c.level)); + FleetHealth { overall, checks } + } +} + +/// Configured free-space thresholds for the disk-space check. +#[derive(Debug, Clone, Copy)] +pub struct DiskThresholds { + /// Evict a node once free space falls to/below this many bytes. + pub eviction_bytes: u64, + /// Turn the health to `Warning` once free space falls to/below this many bytes. + pub warning_bytes: u64, +} + +impl Default for DiskThresholds { + fn default() -> Self { + DiskThresholds { + eviction_bytes: EVICTION_THRESHOLD_MB * MIB, + warning_bytes: WARNING_THRESHOLD_MB * MIB, + } + } +} + +/// Evaluate the disk-space health of a single partition. +fn disk_space_check(p: &PartitionState, thresholds: &DiskThresholds) -> HealthCheck { + let available = p.available_bytes; + let candidate = p.eviction_candidate(); + // Eviction only helps the *other* nodes on the partition, so it is only possible when at least + // two nodes share it (one is evicted, at least one remains to benefit). A partition with a + // single node therefore cannot be auto-helped — this subsumes the "only one node running" case. + let can_evict = p.nodes.len() >= 2; + + let (level, summary) = if available <= thresholds.eviction_bytes { + if can_evict { + let who = candidate + .map(|c| format!("node {}", c.node_id)) + .unwrap_or_else(|| "a node".to_string()); + ( + HealthLevel::Critical, + format!( + "Disk space critical on {}: {} free (≤ {}). Evicting {} to reclaim space.", + p.partition, + fmt_bytes(available), + fmt_bytes(thresholds.eviction_bytes), + who, + ), + ) + } else { + ( + HealthLevel::Critical, + format!( + "Disk space critical on {}: {} free (≤ {}), but only one node is running here \ + so it cannot be auto-evicted. Free disk space or reduce node count manually.", + p.partition, + fmt_bytes(available), + fmt_bytes(thresholds.eviction_bytes), + ), + ) + } + } else if available <= thresholds.warning_bytes { + let who = candidate + .filter(|_| can_evict) + .map(|c| format!("; node {} would be evicted next", c.node_id)) + .unwrap_or_default(); + ( + HealthLevel::Warning, + format!( + "Disk space low on {}: {} free. An eviction may occur once it reaches {}{}.", + p.partition, + fmt_bytes(available), + fmt_bytes(thresholds.eviction_bytes), + who, + ), + ) + } else { + ( + HealthLevel::Green, + format!("Disk space healthy on {}: {} free.", p.partition, fmt_bytes(available)), + ) + }; + + // Only surface a candidate when an eviction could actually happen. + let candidate_out = if can_evict && level != HealthLevel::Green { + candidate.map(|c| EvictionCandidate { + node_id: c.node_id, + data_dir: c.data_dir.to_string_lossy().into_owned(), + size_bytes: c.size_bytes, + }) + } else { + None + }; + + HealthCheck { + kind: HealthCheckKind::DiskSpace, + level, + summary, + partition: Some(p.partition.to_string()), + available_bytes: Some(available), + eviction_threshold_bytes: Some(thresholds.eviction_bytes), + candidate: candidate_out, + } +} + +/// Format a byte count as a human-friendly string (GiB/MiB). +fn fmt_bytes(bytes: u64) -> String { + const GIB: u64 = 1024 * MIB; + if bytes >= GIB { + format!("{:.2} GiB", bytes as f64 / GIB as f64) + } else { + format!("{:.0} MiB", bytes as f64 / MIB as f64) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::node::daemon::disk::{NodeDiskUsage, PartitionKey, PartitionState}; + use std::path::PathBuf; + + fn node(id: u32, size: u64) -> NodeDiskUsage { + NodeDiskUsage { + node_id: id, + data_dir: PathBuf::from(format!("/data/node-{id}")), + size_bytes: size, + } + } + + fn partition(available_bytes: u64, nodes: Vec) -> PartitionState { + PartitionState { + partition: PartitionKey::for_test("p0"), + available_bytes, + nodes, + } + } + + fn thresholds() -> DiskThresholds { + DiskThresholds { + eviction_bytes: 500 * MIB, + warning_bytes: 1024 * MIB, + } + } + + #[test] + fn green_when_space_comfortable() { + let p = partition(4 * 1024 * MIB, vec![node(1, 100), node(2, 200)]); + let health = FleetHealth::from_partitions(&[p], &thresholds()); + assert_eq!(health.overall, HealthLevel::Green); + assert_eq!(health.checks[0].candidate, None); + } + + #[test] + fn warning_names_candidate_between_thresholds() { + // 800 MiB free: below warning (1024) but above eviction (500). + let p = partition(800 * MIB, vec![node(1, 900), node(2, 100)]); + let health = FleetHealth::from_partitions(&[p], &thresholds()); + assert_eq!(health.overall, HealthLevel::Warning); + // Smallest node (2) is the candidate. + assert_eq!(health.checks[0].candidate.as_ref().unwrap().node_id, 2); + } + + #[test] + fn critical_when_at_eviction_threshold_multi_node() { + let p = partition(400 * MIB, vec![node(1, 900), node(2, 100)]); + let health = FleetHealth::from_partitions(&[p], &thresholds()); + assert_eq!(health.overall, HealthLevel::Critical); + assert_eq!(health.checks[0].candidate.as_ref().unwrap().node_id, 2); + assert!(health.checks[0].summary.contains("Evicting node 2")); + } + + #[test] + fn critical_sole_node_warns_no_candidate() { + // Single node on a full partition: cannot auto-evict. + let p = partition(400 * MIB, vec![node(7, 100)]); + let health = FleetHealth::from_partitions(&[p], &thresholds()); + assert_eq!(health.overall, HealthLevel::Critical); + assert!(health.checks[0].candidate.is_none()); + assert!(health.checks[0].summary.contains("only one node")); + } + + #[test] + fn overall_is_worst_across_partitions() { + let healthy = partition(4 * 1024 * MIB, vec![node(1, 100), node(2, 100)]); + let warning = partition(800 * MIB, vec![node(3, 100), node(4, 100)]); + let health = FleetHealth::from_partitions(&[healthy, warning], &thresholds()); + assert_eq!(health.overall, HealthLevel::Warning); + assert_eq!(health.checks.len(), 2); + } + + #[test] + fn empty_fleet_is_green() { + let health = FleetHealth::from_partitions(&[], &thresholds()); + assert_eq!(health.overall, HealthLevel::Green); + assert!(health.checks.is_empty()); + } +} diff --git a/ant-core/src/node/daemon/mod.rs b/ant-core/src/node/daemon/mod.rs index 313222ad..6a6bc6c1 100644 --- a/ant-core/src/node/daemon/mod.rs +++ b/ant-core/src/node/daemon/mod.rs @@ -1,3 +1,5 @@ pub mod client; +pub mod disk; +pub mod health; pub mod server; pub mod supervisor; diff --git a/ant-core/src/node/daemon/server.rs b/ant-core/src/node/daemon/server.rs index d139b0f3..26301154 100644 --- a/ant-core/src/node/daemon/server.rs +++ b/ant-core/src/node/daemon/server.rs @@ -15,9 +15,10 @@ use tokio_util::sync::CancellationToken; use crate::error::Result; use crate::node::binary::NoopProgress; +use crate::node::daemon::health::{DiskThresholds, FleetHealth}; use crate::node::daemon::supervisor::{ - spawn_liveness_monitor, spawn_upgrade_monitor, Supervisor, LIVENESS_POLL_INTERVAL, - UPGRADE_POLL_INTERVAL, + spawn_eviction_monitor, spawn_liveness_monitor, spawn_upgrade_monitor, Supervisor, + EVICTION_POLL_INTERVAL, LIVENESS_POLL_INTERVAL, UPGRADE_POLL_INTERVAL, }; use crate::node::events::NodeEvent; use crate::node::registry::NodeRegistry; @@ -36,6 +37,9 @@ pub struct AppState { pub config: DaemonConfig, /// The actual address the server bound to (resolves port 0 to real port). pub bound_port: u16, + /// Latest fleet health snapshot, refreshed by the eviction monitor and served at + /// `GET /api/v1/health`. + pub health: Arc>, } /// Start the daemon HTTP server. @@ -82,6 +86,8 @@ pub async fn start( } } + let health = Arc::new(RwLock::new(FleetHealth::healthy())); + let state = Arc::new(AppState { registry: registry.clone(), supervisor: supervisor.clone(), @@ -89,6 +95,7 @@ pub async fn start( start_time: Instant::now(), config: config.clone(), bound_port: bound_addr.port(), + health: health.clone(), }); // Background task: probe each Running node's on-disk binary for version drift caused by @@ -101,6 +108,20 @@ pub async fn start( shutdown.clone(), ); + // Background task: monitor free disk space at node data directories. Refreshes the fleet health + // snapshot every tick and auto-evicts a node (smallest data dir) on any partition that has + // fallen to the eviction threshold while ≥2 nodes remain. The threshold is a fixed internal + // constant (mirroring ant-node's own refuse-to-store reserve), not user-configurable. + spawn_eviction_monitor( + registry.clone(), + supervisor.clone(), + event_tx.clone(), + health, + DiskThresholds::default(), + EVICTION_POLL_INTERVAL, + shutdown.clone(), + ); + // Background task: poll adopted nodes' PIDs for OS liveness. Daemon-spawned nodes // get exit detection via `monitor_node`'s owned `Child` handle; adopted nodes don't, // so this poll is the only way the supervisor learns when one of them exits. @@ -151,6 +172,7 @@ fn build_router(state: Arc) -> Router { Router::new() .route("/console", get(get_console)) .route("/api/v1/status", get(get_status)) + .route("/api/v1/health", get(get_health)) .route("/api/v1/events", get(get_events)) .route("/api/v1/nodes/status", get(get_nodes_status)) .route("/api/v1/nodes", post(post_nodes)) @@ -185,6 +207,13 @@ async fn get_status(State(state): State>) -> Json { }) } +/// GET /api/v1/health — Current fleet health (overall level + per-check findings). +/// +/// Refreshed by the eviction monitor; reflects disk pressure and the next eviction candidate. +async fn get_health(State(state): State>) -> Json { + Json(state.health.read().await.clone()) +} + async fn get_events( State(state): State>, ) -> Sse>> { @@ -218,9 +247,15 @@ async fn get_nodes_status(State(state): State>) -> Json { @@ -229,9 +264,15 @@ async fn get_nodes_status(State(state): State>) -> Json total_stopped += 1, } - let pid = supervisor.node_pid(config.id); - let uptime_secs = supervisor.node_uptime_secs(config.id); - let pending_version = supervisor.node_pending_version(config.id); + let (pid, uptime_secs, pending_version) = if config.eviction.is_some() { + (None, None, None) + } else { + ( + supervisor.node_pid(config.id), + supervisor.node_uptime_secs(config.id), + supervisor.node_pending_version(config.id), + ) + }; nodes.push(NodeStatusSummary { node_id: config.id, @@ -241,6 +282,7 @@ async fn get_nodes_status(State(state): State>) -> Json Result { let node_id = config.id; + // An evicted node's data directory has been deleted; it must not be restarted. Recovery is + // to dismiss it (remove from the registry) and add a fresh node. + if config.eviction.is_some() { + return Err(Error::NodeEvicted(node_id)); + } + if let Some(state) = self.node_states.get(&node_id) { if state.status == NodeStatus::Running { return Err(Error::NodeAlreadyRunning(node_id)); @@ -478,7 +494,8 @@ impl Supervisor { NodeStatus::Running | NodeStatus::Starting | NodeStatus::UpgradeScheduled => { running += 1 } - NodeStatus::Stopped | NodeStatus::Stopping => stopped += 1, + // An evicted node is not running; count it alongside stopped for these totals. + NodeStatus::Stopped | NodeStatus::Stopping | NodeStatus::Evicted => stopped += 1, NodeStatus::Errored => errored += 1, } } @@ -687,6 +704,207 @@ pub fn spawn_upgrade_monitor( }); } +/// Background task: monitor free disk space at each node's data directory and, when a partition +/// falls to its eviction threshold, automatically evict a node to reclaim space. +/// +/// Each tick it (1) measures every running node's data directory grouped by partition, (2) refreshes +/// the shared [`FleetHealth`] snapshot so the CLI/GUI can show how close the fleet is to an +/// eviction, and (3) evicts the selected candidate on any partition that is at/below the threshold +/// *and* still has at least two nodes (so a node remains to benefit). Eviction stops the process, +/// deletes the data directory, records a persisted [`EvictionRecord`], and emits an event. It +/// re-measures and may evict again — bounded by [`MAX_EVICTIONS_PER_CYCLE`] — because a single +/// eviction may not free enough on a heavily over-provisioned partition. +/// +/// The task exits when `shutdown` is cancelled. +pub fn spawn_eviction_monitor( + registry: Arc>, + supervisor: Arc>, + event_tx: broadcast::Sender, + health: Arc>, + thresholds: DiskThresholds, + interval: Duration, + shutdown: CancellationToken, +) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + // Skip the immediate first tick so we don't evict while nodes are still starting up. + ticker.tick().await; + + loop { + tokio::select! { + _ = shutdown.cancelled() => return, + _ = ticker.tick() => {}, + } + + run_eviction_cycle(®istry, &supervisor, &event_tx, &health, &thresholds).await; + } + }); +} + +/// Run one disk-pressure check: evict as needed (bounded), then refresh the health snapshot. +async fn run_eviction_cycle( + registry: &Arc>, + supervisor: &Arc>, + event_tx: &broadcast::Sender, + health: &Arc>, + thresholds: &DiskThresholds, +) { + for _ in 0..MAX_EVICTIONS_PER_CYCLE { + let partitions = disk::partition_states(running_nodes(registry, supervisor).await); + + // A partition needs an eviction when it is at/below the threshold and has a spare node to + // sacrifice (≥2 nodes, so one remains). The sole-node case is deliberately left for the + // health layer to surface as Critical rather than auto-evicting the only node. + let target = partitions.iter().find(|p| { + p.available_bytes <= thresholds.eviction_bytes && p.nodes.len() >= 2 + }); + + let Some(partition) = target else { + // Nothing more to evict: publish the current health and finish this cycle. + publish_health(health, event_tx, FleetHealth::from_partitions(&partitions, thresholds)) + .await; + return; + }; + + let Some(candidate) = partition.eviction_candidate().cloned() else { + break; + }; + + evict_node(registry, supervisor, event_tx, &candidate, partition.available_bytes).await; + } + + // Reached the per-cycle eviction cap (or hit a candidate-less partition): refresh health so the + // snapshot reflects reality before the next tick. + let partitions = disk::partition_states(running_nodes(registry, supervisor).await); + publish_health(health, event_tx, FleetHealth::from_partitions(&partitions, thresholds)).await; +} + +/// Snapshot of currently-running, non-evicted nodes as `(id, data_dir)` pairs. +async fn running_nodes( + registry: &Arc>, + supervisor: &Arc>, +) -> Vec<(u32, PathBuf)> { + let reg = registry.read().await; + let sup = supervisor.read().await; + reg.list() + .into_iter() + .filter(|config| config.eviction.is_none()) + .filter(|config| matches!(sup.node_status(config.id), Ok(NodeStatus::Running))) + .map(|config| (config.id, config.data_dir.clone())) + .collect() +} + +/// Evict a single node: stop it, delete its data directory, persist the eviction marker, mark its +/// runtime state, and emit an event. Best-effort — individual failures are logged but do not abort +/// the wider cycle, since leaving a half-evicted node is worse than continuing. +async fn evict_node( + registry: &Arc>, + supervisor: &Arc>, + event_tx: &broadcast::Sender, + candidate: &disk::NodeDiskUsage, + available_before: u64, +) { + let node_id = candidate.node_id; + + // 1. Stop the process. The monitor_node task sees the Stopping/Stopped transition and will not + // respawn it. + if let Err(e) = supervisor.write().await.stop_node(node_id).await { + tracing::warn!("Eviction: failed to stop node {node_id} before deletion: {e}"); + } + + // 2. Delete the data directory — this is what actually reclaims disk space. + let reclaimed = candidate.size_bytes; + if let Err(e) = std::fs::remove_dir_all(&candidate.data_dir) { + // If the directory is already gone that's fine; otherwise warn but still record the + // eviction so the node doesn't keep being re-selected. + if candidate.data_dir.exists() { + tracing::warn!( + "Eviction: failed to delete data dir {} for node {node_id}: {e}", + candidate.data_dir.display() + ); + } + } + + let reason = format!( + "Automatically evicted to reclaim disk space: only {} free on its partition. \ + Its data directory was deleted, recovering ~{}.", + fmt_bytes(available_before), + fmt_bytes(reclaimed), + ); + + // 3. Persist the eviction marker so the `Evicted` status survives daemon restarts. + { + let mut reg = registry.write().await; + if let Ok(config) = reg.get_mut(node_id) { + config.eviction = Some(EvictionRecord { + reason: reason.clone(), + evicted_at: now_unix_secs(), + reclaimed_bytes: reclaimed, + }); + } + if let Err(e) = reg.save() { + tracing::error!("Eviction: failed to persist registry after evicting node {node_id}: {e}"); + } + } + + // 4. Reflect the terminal state in the supervisor's runtime view (status queries also derive + // Evicted from the marker, so this is belt-and-suspenders for any in-memory reader). + supervisor + .write() + .await + .update_state(node_id, NodeStatus::Evicted, None); + + tracing::info!("Evicted node {node_id}, reclaimed ~{} ({reason})", fmt_bytes(reclaimed)); + let _ = event_tx.send(NodeEvent::NodeEvicted { + node_id, + reason, + reclaimed_bytes: reclaimed, + }); +} + +/// Store the new health snapshot, emitting a `FleetHealthChanged` event if the overall level moved. +async fn publish_health( + health: &Arc>, + event_tx: &broadcast::Sender, + next: FleetHealth, +) { + let changed = { + let mut current = health.write().await; + let changed = current.overall != next.overall; + *current = next.clone(); + changed + }; + if changed { + let _ = event_tx.send(NodeEvent::FleetHealthChanged { + overall: serde_json::to_value(next.overall) + .ok() + .and_then(|v| v.as_str().map(str::to_owned)) + .unwrap_or_default(), + }); + } +} + +/// Current Unix time in whole seconds. Falls back to 0 if the clock is before the epoch. +fn now_unix_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// Format a byte count as a human-friendly string (GiB/MiB), matching the health layer's style. +fn fmt_bytes(bytes: u64) -> String { + const MIB: f64 = 1024.0 * 1024.0; + const GIB: f64 = 1024.0 * MIB; + let b = bytes as f64; + if b >= GIB { + format!("{:.2} GiB", b / GIB) + } else { + format!("{:.0} MiB", b / MIB) + } +} + /// Build CLI arguments for the node binary from a NodeConfig. pub fn build_node_args(config: &NodeConfig) -> Vec { let mut args = vec![ @@ -786,7 +1004,11 @@ async fn monitor_node_inner( }; match status_at_exit { - Some(NodeStatus::Stopped) | Some(NodeStatus::Stopping) => return, + // Stopped/Stopping are intentional; Evicted means the daemon deleted the data dir to + // reclaim space. In all three cases the node must not be respawned. + Some(NodeStatus::Stopped) | Some(NodeStatus::Stopping) | Some(NodeStatus::Evicted) => { + return + } Some(NodeStatus::UpgradeScheduled) => { // ant-node cleanly exited after replacing its binary in place. Respawn // directly (no backoff, no crash counter) and refresh the recorded version. @@ -1296,6 +1518,7 @@ mod tests { bootstrap_peers: vec!["peer1".to_string(), "peer2".to_string()], upgrade_channel: None, evm_network: EvmNetwork::default(), + eviction: None, }; let args = build_node_args(&config); @@ -1340,6 +1563,7 @@ mod tests { bootstrap_peers: vec![], upgrade_channel: None, evm_network: EvmNetwork::ArbitrumSepolia, + eviction: None, }; let args = build_node_args(&config); @@ -1365,6 +1589,7 @@ mod tests { bootstrap_peers: vec![], upgrade_channel: Some(UpgradeChannel::Beta), evm_network: EvmNetwork::default(), + eviction: None, }; let args = build_node_args(&config); @@ -1395,6 +1620,7 @@ mod tests { bootstrap_peers: vec![], upgrade_channel: None, evm_network: EvmNetwork::default(), + eviction: None, }; let args = build_node_args(&config); diff --git a/ant-core/src/node/events.rs b/ant-core/src/node/events.rs index 4b2de630..3c91523a 100644 --- a/ant-core/src/node/events.rs +++ b/ant-core/src/node/events.rs @@ -57,6 +57,21 @@ pub enum NodeEvent { old_version: String, new_version: String, }, + /// Emitted when the daemon automatically evicts a node to reclaim disk space: its process was + /// stopped and its data directory deleted. The node now reports as `Evicted` until dismissed. + NodeEvicted { + node_id: u32, + /// Human-readable explanation of the eviction. + reason: String, + /// Approximate bytes reclaimed by deleting the node's data directory. + reclaimed_bytes: u64, + }, + /// Emitted when the fleet's overall health level changes (e.g. green → warning as disk fills), + /// so a connected GUI can refresh its always-visible health indicator without polling. + FleetHealthChanged { + /// Snake-case overall level: `green` | `warning` | `critical`. + overall: String, + }, } impl NodeEvent { @@ -75,6 +90,8 @@ impl NodeEvent { NodeEvent::DownloadComplete { .. } => "download_complete", NodeEvent::UpgradeScheduled { .. } => "upgrade_scheduled", NodeEvent::NodeUpgraded { .. } => "node_upgraded", + NodeEvent::NodeEvicted { .. } => "node_evicted", + NodeEvent::FleetHealthChanged { .. } => "fleet_health_changed", } } } diff --git a/ant-core/src/node/mod.rs b/ant-core/src/node/mod.rs index 1aa07117..26ea4c31 100644 --- a/ant-core/src/node/mod.rs +++ b/ant-core/src/node/mod.rs @@ -98,6 +98,7 @@ pub async fn add_nodes( bootstrap_peers: opts.bootstrap_peers.clone(), upgrade_channel: opts.upgrade_channel, evm_network: opts.evm_network, + eviction: None, }; let assigned_id = registry.add(config); @@ -192,20 +193,30 @@ pub fn reset(registry_path: &Path) -> Result { /// Get the status of all registered nodes without the daemon. /// -/// Since the daemon is not running, all nodes are reported as `Stopped`. +/// Since the daemon is not running, nodes are reported as `Stopped` — except those carrying a +/// persisted [`EvictionRecord`], which are reported as `Evicted` (the marker survives independently +/// of the daemon). pub fn node_status_offline(registry_path: &Path) -> Result { let registry = NodeRegistry::load(registry_path)?; let nodes: Vec = registry .list() .iter() - .map(|config| NodeStatusSummary { - node_id: config.id, - name: config.service_name.clone(), - version: config.version.clone(), - status: NodeStatus::Stopped, - pid: None, - uptime_secs: None, - pending_version: None, + .map(|config| { + let status = if config.eviction.is_some() { + NodeStatus::Evicted + } else { + NodeStatus::Stopped + }; + NodeStatusSummary { + node_id: config.id, + name: config.service_name.clone(), + version: config.version.clone(), + status, + pid: None, + uptime_secs: None, + pending_version: None, + eviction: config.eviction.clone(), + } }) .collect(); let total_stopped = nodes.len() as u32; @@ -484,6 +495,7 @@ mod tests { bootstrap_peers: vec![], upgrade_channel: None, evm_network: EvmNetwork::default(), + eviction: None, }); registry.save().unwrap(); drop(_lock); @@ -568,6 +580,7 @@ mod tests { bootstrap_peers: vec![], upgrade_channel: None, evm_network: EvmNetwork::default(), + eviction: None, }); registry.add(NodeConfig { id: 0, @@ -582,6 +595,7 @@ mod tests { bootstrap_peers: vec![], upgrade_channel: None, evm_network: EvmNetwork::default(), + eviction: None, }); registry.save().unwrap(); drop(_lock); @@ -606,6 +620,47 @@ mod tests { assert_eq!(result.total_stopped, 0); } + #[test] + fn node_status_offline_reports_evicted_from_marker() { + use crate::node::types::EvictionRecord; + + let tmp = tempfile::tempdir().unwrap(); + let reg_path = test_registry_path(tmp.path()); + + let (mut registry, _lock) = NodeRegistry::load_locked(®_path).unwrap(); + registry.add(NodeConfig { + id: 0, + service_name: String::new(), + rewards_address: "0xtest".to_string(), + data_dir: PathBuf::from("/tmp/test-evicted"), + log_dir: None, + node_port: None, + binary_path: PathBuf::from("/usr/bin/antnode"), + version: "0.110.0".to_string(), + env_variables: HashMap::new(), + bootstrap_peers: vec![], + upgrade_channel: None, + evm_network: EvmNetwork::default(), + eviction: Some(EvictionRecord { + reason: "Low disk space".to_string(), + evicted_at: 1_700_000_000, + reclaimed_bytes: 1_000_000, + }), + }); + registry.save().unwrap(); + drop(_lock); + + let result = node_status_offline(®_path).unwrap(); + assert_eq!(result.nodes.len(), 1); + // The persisted marker makes the node report as Evicted even with the daemon down, + // and carries its supplementary reason text through to the summary. + assert_eq!(result.nodes[0].status, NodeStatus::Evicted); + assert_eq!( + result.nodes[0].eviction.as_ref().unwrap().reason, + "Low disk space" + ); + } + #[test] fn reset_empty_registry_succeeds() { let tmp = tempfile::tempdir().unwrap(); diff --git a/ant-core/src/node/registry.rs b/ant-core/src/node/registry.rs index c4590acb..13208df2 100644 --- a/ant-core/src/node/registry.rs +++ b/ant-core/src/node/registry.rs @@ -150,6 +150,7 @@ mod tests { bootstrap_peers: vec![], upgrade_channel: None, evm_network: EvmNetwork::default(), + eviction: None, } } diff --git a/ant-core/src/node/types.rs b/ant-core/src/node/types.rs index 77c30a85..56fadb28 100644 --- a/ant-core/src/node/types.rs +++ b/ant-core/src/node/types.rs @@ -74,6 +74,28 @@ pub enum NodeStatus { /// yet restarted. The supervisor is waiting for the current process to exit and will then /// respawn it against the new binary. UpgradeScheduled, + /// The daemon automatically stopped this node and deleted its data directory to reclaim disk + /// space for the remaining nodes. This is a terminal state derived from a persisted + /// [`EvictionRecord`] on the node's config — it survives daemon restarts and is cleared only + /// when the user dismisses the node (which removes it from the registry entirely). + Evicted, +} + +/// Record of an automatic eviction, persisted on a node's [`NodeConfig`] so that the `evicted` +/// state survives daemon restarts. +/// +/// [`NodeStatus`] is otherwise runtime-only (rebuilt from process scans on startup), so the +/// presence of this record — not live process state — is what makes a node report as +/// [`NodeStatus::Evicted`]. Eviction deletes the node's data directory but keeps the registry +/// entry until the user dismisses it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct EvictionRecord { + /// Human-readable explanation of why the node was evicted (shown to the user). + pub reason: String, + /// Unix epoch seconds at which the eviction occurred. + pub evicted_at: u64, + /// Approximate number of bytes reclaimed by deleting the node's data directory. + pub reclaimed_bytes: u64, } /// Persisted configuration for a single node. @@ -98,6 +120,12 @@ pub struct NodeConfig { /// EVM network the node uses for storage payments. #[serde(default)] pub evm_network: EvmNetwork, + /// Set when the daemon has automatically evicted this node to reclaim disk space. Its presence + /// makes the node report as [`NodeStatus::Evicted`] regardless of runtime state, and persists + /// across daemon restarts. `None` for normal nodes. Backwards compatible: defaults to `None` + /// when absent from an older registry file. + #[serde(default)] + pub eviction: Option, } /// Runtime information for a running node (held in daemon memory only). @@ -424,6 +452,10 @@ pub struct NodeStatusSummary { /// binary reports. Omitted otherwise. #[serde(skip_serializing_if = "Option::is_none")] pub pending_version: Option, + /// Set only when `status == Evicted`: details of why/when the node was evicted, so the CLI and + /// GUI can show supplementary text. Omitted otherwise. + #[serde(skip_serializing_if = "Option::is_none")] + pub eviction: Option, } /// Result of querying node status across all registered nodes. @@ -480,6 +512,59 @@ mod tests { assert_eq!(parsed, NodeStatus::UpgradeScheduled); } + #[test] + fn node_status_evicted_serializes() { + let json = serde_json::to_string(&NodeStatus::Evicted).unwrap(); + assert_eq!(json, "\"evicted\""); + let parsed: NodeStatus = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, NodeStatus::Evicted); + } + + #[test] + fn node_config_eviction_defaults_to_none_for_old_registry() { + // A registry entry written before the eviction field existed must still deserialize. + let legacy = r#"{ + "id": 1, + "service_name": "antnode-1", + "rewards_address": "0xabc", + "data_dir": "/data/node-1", + "log_dir": null, + "node_port": null, + "metrics_port": null, + "network_id": 1, + "binary_path": "/bin/antnode", + "version": "0.1.0", + "env_variables": {}, + "bootstrap_peers": [] + }"#; + let config: NodeConfig = serde_json::from_str(legacy).unwrap(); + assert!(config.eviction.is_none()); + assert!(config.upgrade_channel.is_none()); + } + + #[test] + fn eviction_record_roundtrips_on_summary() { + let summary = NodeStatusSummary { + node_id: 3, + name: "antnode-3".to_string(), + version: "0.1.0".to_string(), + status: NodeStatus::Evicted, + pid: None, + uptime_secs: None, + pending_version: None, + eviction: Some(EvictionRecord { + reason: "Low disk: 480 MiB free, evicting smallest node".to_string(), + evicted_at: 1_700_000_000, + reclaimed_bytes: 2_147_483_648, + }), + }; + let json = serde_json::to_string(&summary).unwrap(); + assert!(json.contains("\"status\":\"evicted\"")); + assert!(json.contains("\"reclaimed_bytes\":2147483648")); + let parsed: NodeStatusSummary = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.eviction.unwrap().evicted_at, 1_700_000_000); + } + #[test] fn node_status_summary_with_pending_version() { let summary = NodeStatusSummary { @@ -490,6 +575,7 @@ mod tests { pid: Some(4242), uptime_secs: Some(3600), pending_version: Some("0.10.11-rc.1".to_string()), + eviction: None, }; let json = serde_json::to_string(&summary).unwrap(); assert!(json.contains("\"status\":\"upgrade_scheduled\"")); @@ -580,6 +666,7 @@ mod tests { pid: Some(1234), uptime_secs: Some(60), pending_version: None, + eviction: None, }, NodeStatusSummary { node_id: 2, @@ -589,6 +676,7 @@ mod tests { pid: None, uptime_secs: None, pending_version: None, + eviction: None, }, ], total_running: 1, @@ -614,6 +702,7 @@ mod tests { pid: Some(5678), uptime_secs: Some(120), pending_version: None, + eviction: None, }; let json = serde_json::to_string(&summary).unwrap(); assert!(json.contains("\"node_id\":1")); @@ -633,6 +722,7 @@ mod tests { pid: None, uptime_secs: None, pending_version: None, + eviction: None, }; let json_stopped = serde_json::to_string(&stopped).unwrap(); assert!(!json_stopped.contains("pid")); From 694ec73f4da07f2ff3c6e38489abb9ad86906c91 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 1 Jul 2026 22:05:53 +0100 Subject: [PATCH 06/22] fix: skip evicted nodes when starting or stopping An evicted node's data directory has been deleted, so trying to start it fails with a confusing "No such file or directory", and `start-all` reports it as a failure. Treat evicted nodes as non-actionable: `start-all`/`stop-all` skip them silently (they stay visible as `Evicted` until dismissed), and single start/stop return a clear message pointing at `ant node dismiss`. Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-core/src/node/daemon/server.rs | 40 +++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/ant-core/src/node/daemon/server.rs b/ant-core/src/node/daemon/server.rs index 26301154..19f629f7 100644 --- a/ant-core/src/node/daemon/server.rs +++ b/ant-core/src/node/daemon/server.rs @@ -412,6 +412,21 @@ async fn post_start_node( }; drop(registry); + // An evicted node's data directory is gone; refuse to start it (recovery is to dismiss and + // re-add, not restart). + if config.eviction.is_some() { + return Err(( + StatusCode::CONFLICT, + Json(serde_json::json!({ + "error": format!( + "Node {id} has been evicted and cannot be started. Dismiss it with \ + `ant node dismiss {id}` and add a new node instead." + ), + "current_state": { "node_id": id, "status": "evicted" } + })), + )); + } + let supervisor_ref = state.supervisor.clone(); // Acquire write lock once for atomic check-and-act (avoids TOCTOU race) @@ -465,7 +480,14 @@ async fn post_start_node( /// POST /api/v1/nodes/start-all — Start all registered nodes. async fn post_start_all(State(state): State>) -> Json { let registry = state.registry.read().await; - let configs: Vec<_> = registry.list().into_iter().cloned().collect(); + // Evicted nodes have no data directory; skip them silently rather than attempting a spawn that + // would fail. They remain visible as `Evicted` in status until dismissed. + let configs: Vec<_> = registry + .list() + .into_iter() + .filter(|c| c.eviction.is_none()) + .cloned() + .collect(); drop(registry); let mut started = Vec::new(); @@ -524,6 +546,20 @@ async fn post_stop_node( }; drop(registry); + // An evicted node is already stopped and its data directory deleted; there is nothing to stop. + if config.eviction.is_some() { + return Err(( + StatusCode::CONFLICT, + Json(serde_json::json!({ + "error": format!( + "Node {id} has been evicted; there is nothing to stop. Dismiss it with \ + `ant node dismiss {id}`." + ), + "current_state": { "node_id": id, "status": "evicted" } + })), + )); + } + // Acquire write lock once for atomic check-and-act (avoids TOCTOU race) let mut supervisor = state.supervisor.write().await; if !supervisor.is_running(id) { @@ -572,9 +608,11 @@ async fn post_stop_node( /// POST /api/v1/nodes/stop-all — Stop all running nodes. async fn post_stop_all(State(state): State>) -> Json { let registry = state.registry.read().await; + // Skip evicted nodes — there is nothing to stop, and they should stay `Evicted` until dismissed. let configs: Vec<(u32, String)> = registry .list() .into_iter() + .filter(|c| c.eviction.is_none()) .map(|c| (c.id, c.service_name.clone())) .collect(); drop(registry); From a9769381d595e390c1ecf46a8791c020f3b18a31 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Wed, 1 Jul 2026 22:05:53 +0100 Subject: [PATCH 07/22] test: cover loading a pre-upgrade registry file Guards backward compatibility of node_registry.json across the evm-network and eviction changes: a registry written by an older daemon (carrying the removed network_id/metrics_port fields and lacking evm_network/eviction) must still load, with the unknown fields ignored and the new ones defaulted. Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-core/src/node/registry.rs | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/ant-core/src/node/registry.rs b/ant-core/src/node/registry.rs index 13208df2..30331820 100644 --- a/ant-core/src/node/registry.rs +++ b/ant-core/src/node/registry.rs @@ -174,6 +174,48 @@ mod tests { assert_eq!(reg.get(id).unwrap().rewards_address, "0xtest"); } + #[test] + fn loads_pre_upgrade_registry_file() { + // A registry written by an older daemon: each node still carries the now-removed + // `network_id`/`metrics_port` fields and lacks `evm_network`/`eviction`. A new daemon must + // load it (unknown fields ignored, new fields defaulted) rather than erroring on upgrade. + let legacy = r#"{ + "schema_version": 1, + "nodes": { + "1": { + "id": 1, + "service_name": "node1", + "rewards_address": "0xabc", + "data_dir": "/data/node-1", + "log_dir": "/logs/node-1", + "node_port": 12000, + "metrics_port": 13000, + "network_id": 2, + "binary_path": "/bin/antnode", + "version": "0.1.0", + "env_variables": {}, + "bootstrap_peers": ["peer1"], + "upgrade_channel": null + } + }, + "next_id": 2 + }"#; + + let tmp = NamedTempFile::new().unwrap(); + let path = tmp.path().with_extension("json"); + std::fs::write(&path, legacy).unwrap(); + + let reg = NodeRegistry::load(&path).unwrap(); + let node = reg.get(1).unwrap(); + // Preserved fields survive the round-trip. + assert_eq!(node.node_port, Some(12000)); + assert_eq!(node.bootstrap_peers, vec!["peer1".to_string()]); + // Removed fields are ignored (no deny_unknown_fields); new fields take their defaults. + assert!(node.eviction.is_none()); + assert_eq!(node.evm_network, crate::node::types::EvmNetwork::default()); + assert_eq!(reg.next_id, 2); + } + #[test] fn save_and_reload() { let tmp = NamedTempFile::new().unwrap(); From 34d8fef049922ffd87c5ebd4fea18e987a7ca1ee Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 2 Jul 2026 23:44:41 +0100 Subject: [PATCH 08/22] fix: retry data-dir deletion during eviction to reclaim space on Windows On Windows a just-killed node can keep its data files open for a moment after exit (its LMDB memory map and its own copied ant-node binary), and antivirus/indexers can grab transient handles, so remove_dir_all fails with "access denied / in use". The eviction then stopped the node without reclaiming any space, and the reason text falsely claimed it was reclaimed. Retry the delete with bounded exponential backoff (~4.5s) so the OS has time to release the handles; on Unix the first attempt still succeeds. If deletion ultimately fails, the node is still marked Evicted (its process is gone) but the reason text and reclaimed_bytes reflect that no space was reclaimed, and the failure is logged at error level. Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-core/src/node/daemon/supervisor.rs | 91 +++++++++++++++++++++----- 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/ant-core/src/node/daemon/supervisor.rs b/ant-core/src/node/daemon/supervisor.rs index 6855471f..12b7a2ba 100644 --- a/ant-core/src/node/daemon/supervisor.rs +++ b/ant-core/src/node/daemon/supervisor.rs @@ -795,6 +795,31 @@ async fn running_nodes( .collect() } +/// Delete a directory tree, retrying briefly to tolerate transient locks. +/// +/// A node we just killed can hold its data files open for a short moment after exit — on Windows +/// especially (its LMDB memory map and its own copied `ant-node` binary), and antivirus/indexers can +/// grab transient handles — so `remove_dir_all` fails with "access denied / in use" until the OS +/// releases them. A bounded exponential backoff gives it time; on Unix the first attempt almost +/// always succeeds. Returns `Ok` on success or if the directory is already gone. +async fn remove_dir_all_with_retry(path: &Path) -> std::io::Result<()> { + const MAX_ATTEMPTS: u32 = 8; + let mut delay = Duration::from_millis(100); + for attempt in 1..=MAX_ATTEMPTS { + match std::fs::remove_dir_all(path) { + Ok(()) => return Ok(()), + // Already gone — nothing left to reclaim; treat as success. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) if attempt == MAX_ATTEMPTS => return Err(e), + Err(_) => { + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(1)); + } + } + } + Ok(()) +} + /// Evict a single node: stop it, delete its data directory, persist the eviction marker, mark its /// runtime state, and emit an event. Best-effort — individual failures are logged but do not abort /// the wider cycle, since leaving a half-evicted node is worse than continuing. @@ -813,25 +838,40 @@ async fn evict_node( tracing::warn!("Eviction: failed to stop node {node_id} before deletion: {e}"); } - // 2. Delete the data directory — this is what actually reclaims disk space. + // 2. Delete the data directory — this is what actually reclaims disk space. A just-killed node + // can briefly hold its files open (LMDB memory map, and its own copied binary in the data + // dir); on Windows `remove_dir_all` then fails with "access denied / in use" until the OS + // releases those handles. Retry with backoff so the reclaim actually happens. let reclaimed = candidate.size_bytes; - if let Err(e) = std::fs::remove_dir_all(&candidate.data_dir) { - // If the directory is already gone that's fine; otherwise warn but still record the - // eviction so the node doesn't keep being re-selected. - if candidate.data_dir.exists() { - tracing::warn!( - "Eviction: failed to delete data dir {} for node {node_id}: {e}", + let deleted = match remove_dir_all_with_retry(&candidate.data_dir).await { + Ok(()) => true, + Err(e) => { + tracing::error!( + "Eviction: could not delete data dir {} for node {node_id} after retries: {e}. \ + Disk space was NOT reclaimed; manual cleanup may be required.", candidate.data_dir.display() ); + false } - } - - let reason = format!( - "Automatically evicted to reclaim disk space: only {} free on its partition. \ - Its data directory was deleted, recovering ~{}.", - fmt_bytes(available_before), - fmt_bytes(reclaimed), - ); + }; + let reclaimed_bytes = if deleted { reclaimed } else { 0 }; + + let reason = if deleted { + format!( + "Automatically evicted to reclaim disk space: only {} free on its partition. \ + Its data directory was deleted, recovering ~{}.", + fmt_bytes(available_before), + fmt_bytes(reclaimed), + ) + } else { + format!( + "Automatically evicted due to low disk space (only {} free on its partition), but its \ + data directory could not be deleted, so space was not reclaimed. Manual cleanup of {} \ + may be needed.", + fmt_bytes(available_before), + candidate.data_dir.display(), + ) + }; // 3. Persist the eviction marker so the `Evicted` status survives daemon restarts. { @@ -840,7 +880,7 @@ async fn evict_node( config.eviction = Some(EvictionRecord { reason: reason.clone(), evicted_at: now_unix_secs(), - reclaimed_bytes: reclaimed, + reclaimed_bytes, }); } if let Err(e) = reg.save() { @@ -855,11 +895,11 @@ async fn evict_node( .await .update_state(node_id, NodeStatus::Evicted, None); - tracing::info!("Evicted node {node_id}, reclaimed ~{} ({reason})", fmt_bytes(reclaimed)); + tracing::info!("Evicted node {node_id}, reclaimed ~{} ({reason})", fmt_bytes(reclaimed_bytes)); let _ = event_tx.send(NodeEvent::NodeEvicted { node_id, reason, - reclaimed_bytes: reclaimed, + reclaimed_bytes, }); } @@ -1464,6 +1504,21 @@ mod tests { use super::*; use crate::node::types::{EvmNetwork, UpgradeChannel}; + #[tokio::test] + async fn remove_dir_all_with_retry_deletes_tree_and_tolerates_missing() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("node-data"); + std::fs::create_dir_all(dir.join("sub")).unwrap(); + std::fs::write(dir.join("sub").join("data.mdb"), vec![0u8; 128]).unwrap(); + + // Deletes an existing tree. + remove_dir_all_with_retry(&dir).await.unwrap(); + assert!(!dir.exists()); + + // Idempotent: an already-gone path is treated as success (no error). + remove_dir_all_with_retry(&dir).await.unwrap(); + } + #[test] fn adopted_flag_lifecycle() { let (tx, _rx) = broadcast::channel(16); From 7bbb49eaf6c702550a7d73b9d4f0876a1fab11b6 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 3 Jul 2026 01:38:43 +0100 Subject: [PATCH 09/22] fix: address eviction-safety review Addresses the review on #138. - Eviction candidate sizing (disk.rs): rank and report by actual reclaimable disk usage. dir_size now sums allocated blocks (blocks() * 512 on Unix) instead of logical file length, so LMDB's sparse data.mdb no longer mis-ranks the "smallest" node or misreports reclaimed bytes. It also uses symlink_metadata and skips symlinks so the walk cannot escape the node tree via a link. Adds regression tests for sparse sizing and symlink skipping. - Start/evict race (supervisor.rs): a concurrent POST /nodes/{id}/start could spawn a node during its stop/delete window. evict_node now makes the node unstartable *before* that window, via an in-memory `evicting` flag set under the supervisor lock (atomically excluding start_node) and a persisted eviction marker written first (which also survives a daemon crash mid-delete). Rollback is intentionally not attempted: once stopped, the node stays Evicted. Adds a regression test. - CI: apply rustfmt and fix two rustdoc intra-doc links (a private-item link and an unresolved EvictionRecord link). The security-audit failure (quinn-proto RUSTSEC-2026-0185 via saorsa-transport) and the is_running/UpgradeScheduled observation are pre-existing on main and out of scope here, per the review. Co-Authored-By: Claude Opus 4.8 (1M context) --- ant-cli/src/commands/node/status.rs | 6 +- ant-core/src/node/daemon/disk.rs | 104 ++++++++++-- ant-core/src/node/daemon/health.rs | 6 +- ant-core/src/node/daemon/supervisor.rs | 222 ++++++++++++++++++------- ant-core/src/node/mod.rs | 2 +- 5 files changed, 265 insertions(+), 75 deletions(-) diff --git a/ant-cli/src/commands/node/status.rs b/ant-cli/src/commands/node/status.rs index 8b0e5a56..96178883 100644 --- a/ant-cli/src/commands/node/status.rs +++ b/ant-cli/src/commands/node/status.rs @@ -117,7 +117,11 @@ fn print_fleet_health(health: &FleetHealth) { println!(" {} Fleet health: {}", dot, label); // Surface the reason(s) whenever the fleet is not fully healthy. - for check in health.checks.iter().filter(|c| c.level != HealthLevel::Green) { + for check in health + .checks + .iter() + .filter(|c| c.level != HealthLevel::Green) + { println!(" {} {}", "→".dimmed(), check.summary.dimmed()); } println!(); diff --git a/ant-core/src/node/daemon/disk.rs b/ant-core/src/node/daemon/disk.rs index eb6321f5..9b31ae1b 100644 --- a/ant-core/src/node/daemon/disk.rs +++ b/ant-core/src/node/daemon/disk.rs @@ -71,11 +71,19 @@ impl PartitionState { } } -/// Recursively sum the size of every regular file under `path`. +/// Recursively sum the **allocated** disk usage of every regular file under `path`. /// -/// Symlinks are not followed. Returns 0 if `path` does not exist. Best-effort: entries that cannot -/// be read are skipped rather than aborting the walk, so a transient permission error on one file -/// does not corrupt the eviction decision. +/// Because eviction is automatic and destructive, this must reflect the space that deleting the tree +/// would actually reclaim — not the logical file length. LMDB's `data.mdb` is a sparse file whose +/// logical length is the (often multi-GiB) map size while its allocated blocks are only the bytes it +/// really uses; ranking "smallest data dir" or reporting "reclaimed bytes" off `len()` would be +/// badly wrong. On Unix we therefore use allocated blocks (`blocks() * 512`); elsewhere we fall back +/// to logical length. +/// +/// Symlinks are never followed: entries are examined with `symlink_metadata`, and a symlink is +/// skipped entirely so the walk cannot escape the node's data directory and count unrelated data. +/// Returns 0 if `path` does not exist. Best-effort: entries that cannot be read are skipped rather +/// than aborting the walk, so a transient permission error does not corrupt the eviction decision. pub fn dir_size(path: &Path) -> u64 { let mut total = 0u64; let mut stack = vec![path.to_path_buf()]; @@ -85,22 +93,42 @@ pub fn dir_size(path: &Path) -> u64 { Err(_) => continue, }; for entry in entries.flatten() { - // `symlink_metadata` does not traverse symlinks, so we never double-count or escape the - // tree via a link. - let meta = match entry.metadata() { + // `symlink_metadata` reports the entry itself, never the symlink target. + let meta = match std::fs::symlink_metadata(entry.path()) { Ok(meta) => meta, Err(_) => continue, }; - if meta.is_dir() { + let file_type = meta.file_type(); + if file_type.is_symlink() { + // Do not follow — recursing or counting through a link could escape the tree. + continue; + } else if file_type.is_dir() { stack.push(entry.path()); - } else if meta.is_file() { - total = total.saturating_add(meta.len()); + } else if file_type.is_file() { + total = total.saturating_add(allocated_size(&meta)); } } } total } +/// Disk space a regular file actually occupies. +/// +/// On Unix this is the allocated block count (`blocks() * 512`, per POSIX these are always 512-byte +/// units regardless of filesystem block size), which correctly reflects sparse files. On other +/// platforms std exposes no allocated size, so we fall back to the logical length. +fn allocated_size(meta: &std::fs::Metadata) -> u64 { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + meta.blocks().saturating_mul(512) + } + #[cfg(not(unix))] + { + meta.len() + } +} + /// Free bytes on the partition hosting `path` (or its nearest existing ancestor). /// /// Returns `None` if the available space cannot be determined. @@ -223,11 +251,14 @@ mod tests { #[test] fn dir_size_sums_nested_files() { let tmp = tempfile::tempdir().unwrap(); - std::fs::write(tmp.path().join("a.bin"), vec![0u8; 1000]).unwrap(); + // Non-zero bytes so no filesystem can silently hole-punch them away. + std::fs::write(tmp.path().join("a.bin"), vec![1u8; 1000]).unwrap(); let sub = tmp.path().join("sub"); std::fs::create_dir(&sub).unwrap(); - std::fs::write(sub.join("b.bin"), vec![0u8; 2500]).unwrap(); - assert_eq!(dir_size(tmp.path()), 3500); + std::fs::write(sub.join("b.bin"), vec![1u8; 2500]).unwrap(); + // dir_size reports allocated blocks, which for fully-written files is at least the logical + // 3500 bytes (block-rounded upward). + assert!(dir_size(tmp.path()) >= 3500); } #[test] @@ -235,6 +266,48 @@ mod tests { assert_eq!(dir_size(Path::new("/nonexistent/path/xyz")), 0); } + #[cfg(unix)] + #[test] + fn dir_size_does_not_follow_symlinks() { + // A node data dir containing a symlink to a large external file must not have that file's + // size counted — following it could escape the tree and count unrelated data. + let tmp = tempfile::tempdir().unwrap(); + let node_dir = tmp.path().join("node"); + std::fs::create_dir(&node_dir).unwrap(); + std::fs::write(node_dir.join("real.bin"), vec![1u8; 4000]).unwrap(); + + let external = tmp.path().join("external_big.bin"); + std::fs::write(&external, vec![1u8; 5_000_000]).unwrap(); + std::os::unix::fs::symlink(&external, node_dir.join("link")).unwrap(); + + // Only real.bin is counted; the 5 MB symlink target is not. + let size = dir_size(&node_dir); + assert!(size > 0); + assert!(size < 1_000_000, "symlink target was followed: {size}"); + } + + #[cfg(unix)] + #[test] + fn dir_size_uses_allocated_not_logical_size() { + use std::io::Write; + // A sparse file (LMDB's data.mdb behaves this way): 20 MiB logical, ~0 allocated. + let tmp = tempfile::tempdir().unwrap(); + let f = std::fs::File::create(tmp.path().join("sparse.mdb")).unwrap(); + f.set_len(20 * 1024 * 1024).unwrap(); + drop(f); + // Also a small fully-written file so the dir isn't empty. + let mut real = std::fs::File::create(tmp.path().join("real.bin")).unwrap(); + real.write_all(&[1u8; 2000]).unwrap(); + drop(real); + + // Allocated usage is far below the 20 MiB logical size of the sparse file. + let size = dir_size(tmp.path()); + assert!( + size < 1024 * 1024, + "expected allocated size well under 1 MiB, got {size}" + ); + } + #[test] fn partition_states_groups_colocated_nodes() { // Two nodes under the same temp dir share a partition. @@ -243,8 +316,9 @@ mod tests { let d2 = tmp.path().join("node-2"); std::fs::create_dir_all(&d1).unwrap(); std::fs::create_dir_all(&d2).unwrap(); - std::fs::write(d1.join("data"), vec![0u8; 4000]).unwrap(); - std::fs::write(d2.join("data"), vec![0u8; 1000]).unwrap(); + // Clearly different allocated sizes (non-zero bytes) so selection is by size, not tie-break. + std::fs::write(d1.join("data"), vec![1u8; 200_000]).unwrap(); + std::fs::write(d2.join("data"), vec![1u8; 1000]).unwrap(); let states = partition_states(vec![(1, d1), (2, d2)]); assert_eq!(states.len(), 1, "co-located nodes share one partition"); diff --git a/ant-core/src/node/daemon/health.rs b/ant-core/src/node/daemon/health.rs index fc7f8c61..8f83089a 100644 --- a/ant-core/src/node/daemon/health.rs +++ b/ant-core/src/node/daemon/health.rs @@ -197,7 +197,11 @@ fn disk_space_check(p: &PartitionState, thresholds: &DiskThresholds) -> HealthCh } else { ( HealthLevel::Green, - format!("Disk space healthy on {}: {} free.", p.partition, fmt_bytes(available)), + format!( + "Disk space healthy on {}: {} free.", + p.partition, + fmt_bytes(available) + ), ) }; diff --git a/ant-core/src/node/daemon/supervisor.rs b/ant-core/src/node/daemon/supervisor.rs index 12b7a2ba..7df09deb 100644 --- a/ant-core/src/node/daemon/supervisor.rs +++ b/ant-core/src/node/daemon/supervisor.rs @@ -15,7 +15,8 @@ use crate::node::events::NodeEvent; use crate::node::process::spawn::spawn_node; use crate::node::registry::NodeRegistry; use crate::node::types::{ - EvictionRecord, NodeConfig, NodeStarted, NodeStatus, NodeStopFailed, NodeStopped, StopNodeResult, + EvictionRecord, NodeConfig, NodeStarted, NodeStatus, NodeStopFailed, NodeStopped, + StopNodeResult, }; /// How often the upgrade-detection task polls each running node's binary for a version change. @@ -209,6 +210,10 @@ pub struct Supervisor { /// auto-upgrade, respawn for these nodes happen in the liveness monitor instead. A node /// leaves this set once this daemon (re)spawns it and owns a `monitor_node` for it. adopted: HashSet, + /// Nodes currently being evicted (stop + data-dir delete in progress). Checked and set under + /// the supervisor write lock, so it atomically excludes a concurrent `start_node` for the whole + /// stop/delete window — before which the persisted eviction marker is not yet visible. + evicting: HashSet, } struct NodeRuntime { @@ -227,6 +232,7 @@ impl Supervisor { event_tx, node_states: HashMap::new(), adopted: HashSet::new(), + evicting: HashSet::new(), } } @@ -236,6 +242,19 @@ impl Supervisor { self.adopted.contains(&node_id) } + /// Mark a node as under eviction so `start_node` refuses it for the whole stop/delete window. + /// Both this and the `start_node` check run under the supervisor write lock, so a concurrent + /// start can never slip in and spawn the node while its data directory is being deleted. + fn begin_evicting(&mut self, node_id: u32) { + self.evicting.insert(node_id); + } + + /// Clear the eviction-in-progress flag (the persisted `eviction` marker keeps the node + /// unstartable afterwards). + fn finish_evicting(&mut self, node_id: u32) { + self.evicting.remove(&node_id); + } + /// Mark a node as owned by this daemon (i.e. it now has a `monitor_node` task). Clears /// any adopted flag so the liveness monitor leaves its exit handling to `monitor_node`. fn mark_owned(&mut self, node_id: u32) { @@ -255,8 +274,10 @@ impl Supervisor { let node_id = config.id; // An evicted node's data directory has been deleted; it must not be restarted. Recovery is - // to dismiss it (remove from the registry) and add a fresh node. - if config.eviction.is_some() { + // to dismiss it (remove from the registry) and add a fresh node. The persisted marker covers + // the settled case; the in-progress `evicting` set (set under this same lock) covers the + // window where the delete is still running and the marker's config may not be visible yet. + if config.eviction.is_some() || self.evicting.contains(&node_id) { return Err(Error::NodeEvicted(node_id)); } @@ -712,7 +733,7 @@ pub fn spawn_upgrade_monitor( /// eviction, and (3) evicts the selected candidate on any partition that is at/below the threshold /// *and* still has at least two nodes (so a node remains to benefit). Eviction stops the process, /// deletes the data directory, records a persisted [`EvictionRecord`], and emits an event. It -/// re-measures and may evict again — bounded by [`MAX_EVICTIONS_PER_CYCLE`] — because a single +/// re-measures and may evict again — bounded by `MAX_EVICTIONS_PER_CYCLE` — because a single /// eviction may not free enough on a heavily over-provisioned partition. /// /// The task exits when `shutdown` is cancelled. @@ -756,14 +777,18 @@ async fn run_eviction_cycle( // A partition needs an eviction when it is at/below the threshold and has a spare node to // sacrifice (≥2 nodes, so one remains). The sole-node case is deliberately left for the // health layer to surface as Critical rather than auto-evicting the only node. - let target = partitions.iter().find(|p| { - p.available_bytes <= thresholds.eviction_bytes && p.nodes.len() >= 2 - }); + let target = partitions + .iter() + .find(|p| p.available_bytes <= thresholds.eviction_bytes && p.nodes.len() >= 2); let Some(partition) = target else { // Nothing more to evict: publish the current health and finish this cycle. - publish_health(health, event_tx, FleetHealth::from_partitions(&partitions, thresholds)) - .await; + publish_health( + health, + event_tx, + FleetHealth::from_partitions(&partitions, thresholds), + ) + .await; return; }; @@ -771,13 +796,25 @@ async fn run_eviction_cycle( break; }; - evict_node(registry, supervisor, event_tx, &candidate, partition.available_bytes).await; + evict_node( + registry, + supervisor, + event_tx, + &candidate, + partition.available_bytes, + ) + .await; } // Reached the per-cycle eviction cap (or hit a candidate-less partition): refresh health so the // snapshot reflects reality before the next tick. let partitions = disk::partition_states(running_nodes(registry, supervisor).await); - publish_health(health, event_tx, FleetHealth::from_partitions(&partitions, thresholds)).await; + publish_health( + health, + event_tx, + FleetHealth::from_partitions(&partitions, thresholds), + ) + .await; } /// Snapshot of currently-running, non-evicted nodes as `(id, data_dir)` pairs. @@ -820,9 +857,35 @@ async fn remove_dir_all_with_retry(path: &Path) -> std::io::Result<()> { Ok(()) } -/// Evict a single node: stop it, delete its data directory, persist the eviction marker, mark its -/// runtime state, and emit an event. Best-effort — individual failures are logged but do not abort -/// the wider cycle, since leaving a half-evicted node is worse than continuing. +/// Write (or overwrite) a node's persisted eviction marker and save the registry. +async fn persist_eviction_marker( + registry: &Arc>, + node_id: u32, + reason: &str, + evicted_at: u64, + reclaimed_bytes: u64, +) { + let mut reg = registry.write().await; + if let Ok(config) = reg.get_mut(node_id) { + config.eviction = Some(EvictionRecord { + reason: reason.to_string(), + evicted_at, + reclaimed_bytes, + }); + } + if let Err(e) = reg.save() { + tracing::error!("Eviction: failed to persist registry for node {node_id}: {e}"); + } +} + +/// Evict a single node: make it unstartable, stop it, delete its data directory, finalise the +/// persisted marker, and emit an event. +/// +/// Ordering matters for safety. The node is made unstartable *before* the stop/delete window — via +/// the in-memory `evicting` flag (set under the supervisor lock, so it atomically excludes a +/// concurrent `start_node`) and a persisted eviction marker (which also survives a daemon crash +/// mid-delete). Only then do we stop and delete. Rollback is deliberately not attempted: once the +/// process is stopped the node stays `Evicted` (terminal) whether or not the delete succeeds. async fn evict_node( registry: &Arc>, supervisor: &Arc>, @@ -831,18 +894,28 @@ async fn evict_node( available_before: u64, ) { let node_id = candidate.node_id; - - // 1. Stop the process. The monitor_node task sees the Stopping/Stopped transition and will not - // respawn it. + let reclaimable = candidate.size_bytes; + let evicted_at = now_unix_secs(); + + // 1. Make the node unstartable up front (before we touch the process or its files). + supervisor.write().await.begin_evicting(node_id); + let pending_reason = format!( + "Automatically evicted to reclaim disk space: only {} free on its partition. \ + Deleting its data directory to recover ~{}.", + fmt_bytes(available_before), + fmt_bytes(reclaimable), + ); + persist_eviction_marker(registry, node_id, &pending_reason, evicted_at, reclaimable).await; + + // 2. Stop the process (still marked Running, so this actually kills it). The monitor_node task + // sees the Stopping/Stopped transition and will not respawn it. if let Err(e) = supervisor.write().await.stop_node(node_id).await { tracing::warn!("Eviction: failed to stop node {node_id} before deletion: {e}"); } - // 2. Delete the data directory — this is what actually reclaims disk space. A just-killed node - // can briefly hold its files open (LMDB memory map, and its own copied binary in the data - // dir); on Windows `remove_dir_all` then fails with "access denied / in use" until the OS - // releases those handles. Retry with backoff so the reclaim actually happens. - let reclaimed = candidate.size_bytes; + // 3. Delete the data directory — this reclaims the disk space. A just-killed node can briefly + // hold its files open (LMDB memory map, its own copied binary); on Windows `remove_dir_all` + // then fails until the OS releases the handles, so retry with backoff. let deleted = match remove_dir_all_with_retry(&candidate.data_dir).await { Ok(()) => true, Err(e) => { @@ -854,48 +927,42 @@ async fn evict_node( false } }; - let reclaimed_bytes = if deleted { reclaimed } else { 0 }; - - let reason = if deleted { - format!( - "Automatically evicted to reclaim disk space: only {} free on its partition. \ - Its data directory was deleted, recovering ~{}.", - fmt_bytes(available_before), - fmt_bytes(reclaimed), + + // 4. Finalise the marker to reflect what actually happened, flip runtime state to Evicted, and + // clear the in-progress flag (the persisted marker keeps the node unstartable from here). + let (reclaimed_bytes, reason) = if deleted { + ( + reclaimable, + format!( + "Automatically evicted to reclaim disk space: only {} free on its partition. \ + Its data directory was deleted, recovering ~{}.", + fmt_bytes(available_before), + fmt_bytes(reclaimable), + ), ) } else { - format!( - "Automatically evicted due to low disk space (only {} free on its partition), but its \ - data directory could not be deleted, so space was not reclaimed. Manual cleanup of {} \ - may be needed.", - fmt_bytes(available_before), - candidate.data_dir.display(), + ( + 0, + format!( + "Automatically evicted due to low disk space (only {} free on its partition), but \ + its data directory could not be deleted, so space was not reclaimed. Manual \ + cleanup of {} may be needed.", + fmt_bytes(available_before), + candidate.data_dir.display(), + ), ) }; - - // 3. Persist the eviction marker so the `Evicted` status survives daemon restarts. + persist_eviction_marker(registry, node_id, &reason, evicted_at, reclaimed_bytes).await; { - let mut reg = registry.write().await; - if let Ok(config) = reg.get_mut(node_id) { - config.eviction = Some(EvictionRecord { - reason: reason.clone(), - evicted_at: now_unix_secs(), - reclaimed_bytes, - }); - } - if let Err(e) = reg.save() { - tracing::error!("Eviction: failed to persist registry after evicting node {node_id}: {e}"); - } + let mut sup = supervisor.write().await; + sup.update_state(node_id, NodeStatus::Evicted, None); + sup.finish_evicting(node_id); } - // 4. Reflect the terminal state in the supervisor's runtime view (status queries also derive - // Evicted from the marker, so this is belt-and-suspenders for any in-memory reader). - supervisor - .write() - .await - .update_state(node_id, NodeStatus::Evicted, None); - - tracing::info!("Evicted node {node_id}, reclaimed ~{} ({reason})", fmt_bytes(reclaimed_bytes)); + tracing::info!( + "Evicted node {node_id}, reclaimed ~{} ({reason})", + fmt_bytes(reclaimed_bytes) + ); let _ = event_tx.send(NodeEvent::NodeEvicted { node_id, reason, @@ -1519,6 +1586,47 @@ mod tests { remove_dir_all_with_retry(&dir).await.unwrap(); } + #[tokio::test] + async fn start_node_rejects_a_node_being_evicted() { + let (tx, _rx) = broadcast::channel(16); + let sup = Arc::new(RwLock::new(Supervisor::new(tx))); + + let tmp = tempfile::tempdir().unwrap(); + let reg = Arc::new(RwLock::new( + NodeRegistry::load(&tmp.path().join("reg.json")).unwrap(), + )); + + let config = NodeConfig { + id: 7, + service_name: "node7".to_string(), + rewards_address: "0xabc".to_string(), + data_dir: tmp.path().join("node-7"), + log_dir: None, + node_port: None, + binary_path: "/bin/node".into(), + version: "0.1.0".to_string(), + env_variables: HashMap::new(), + bootstrap_peers: vec![], + upgrade_channel: None, + evm_network: EvmNetwork::default(), + eviction: None, + }; + + // Flagged as evicting -> start is refused before any spawn attempt, closing the race with + // the concurrent stop/delete in evict_node. + sup.write().await.begin_evicting(7); + let res = sup + .write() + .await + .start_node(&config, sup.clone(), reg.clone()) + .await; + assert!(matches!(res, Err(Error::NodeEvicted(7)))); + + // Clearing the flag removes the in-progress guard (the persisted marker takes over). + sup.write().await.finish_evicting(7); + assert!(!sup.read().await.evicting.contains(&7)); + } + #[test] fn adopted_flag_lifecycle() { let (tx, _rx) = broadcast::channel(16); diff --git a/ant-core/src/node/mod.rs b/ant-core/src/node/mod.rs index 26ea4c31..b3113019 100644 --- a/ant-core/src/node/mod.rs +++ b/ant-core/src/node/mod.rs @@ -194,7 +194,7 @@ pub fn reset(registry_path: &Path) -> Result { /// Get the status of all registered nodes without the daemon. /// /// Since the daemon is not running, nodes are reported as `Stopped` — except those carrying a -/// persisted [`EvictionRecord`], which are reported as `Evicted` (the marker survives independently +/// persisted [`EvictionRecord`](crate::node::types::EvictionRecord), which are reported as `Evicted` (the marker survives independently /// of the daemon). pub fn node_status_offline(registry_path: &Path) -> Result { let registry = NodeRegistry::load(registry_path)?; From ee39cb1afd9555c664d772bb7bd05b51cfb57718 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Fri, 3 Jul 2026 14:56:16 +0100 Subject: [PATCH 10/22] chore(release): release ant-client from main -> 0.2.10,0.3.0 --- Cargo.lock | 4 ++-- ant-cli/Cargo.toml | 2 +- ant-core/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bc24539..86093ef3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -815,7 +815,7 @@ dependencies = [ [[package]] name = "ant-cli" -version = "0.2.9" +version = "0.2.10" dependencies = [ "ant-core", "anyhow", @@ -835,7 +835,7 @@ dependencies = [ [[package]] name = "ant-core" -version = "0.2.9" +version = "0.3.0" dependencies = [ "alloy", "ant-node", diff --git a/ant-cli/Cargo.toml b/ant-cli/Cargo.toml index 1aa11db4..e666581a 100644 --- a/ant-cli/Cargo.toml +++ b/ant-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ant-cli" -version = "0.2.9" +version = "0.2.10" edition = "2021" description = "Unified CLI (`ant`) for the Autonomi network: store and retrieve data, and manage local nodes." license = "MIT OR Apache-2.0" diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 09da3272..36973729 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ant-core" -version = "0.2.9" +version = "0.3.0" edition = "2021" description = "Headless Rust library for the Autonomi network: data storage and retrieval with self-encryption and EVM payments, plus node lifecycle management." license = "MIT OR Apache-2.0" From b60c3df887344cbf20df55714f313f9f101d8fe5 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 17 Jun 2026 17:59:44 +0900 Subject: [PATCH 11/22] feat(client): resolve-before-pay for commitment-bound quotes (ADR-0003) Implement the client half of ADR-0003: the client pays nothing it cannot resolve. Before paying, it runs the full binding check on every quote and candidate, so a node that overstates its storage to inflate its price is dropped before any on-chain payment instead of after. For each single-node quote and each merkle candidate, the client now: - verifies the quote's own ML-DSA-65 signature and that it is for the requested content; - checks the (committed_key_count, commitment_pin) shape, the count cap, and that price == calculate_price(count) by exact recomputation; and - for a bound quote, fully resolves the shipped commitment: parse it, check it is bound to the quoting peer, verify its signature, that it hashes to the quote's pin, and that its key_count matches the claimed count. Anything unresolvable, withheld, off-curve, or contradictory is dropped before payment, exactly as the storer would reject it. The verified commitments are then forwarded as sidecars in the PUT bundle (single-node and merkle) so storers cross-check synchronously. Sidecar blobs are size-capped before parsing. Pricing and the commitment verification come from ant-protocol, so client and node never disagree. Adds an e2e test (real QUIC nodes + Anvil) proving bound quotes are shipped, priced, fully resolve, and round-trip; and the off-curve / forged-commitment rejection is covered in unit tests. Depends on the evmlib, ant-protocol, and ant-node ADR-0003 releases; will not build standalone until those publish. --- ant-core/examples/bench-quoting.rs | 2 +- ant-core/src/data/client/batch.rs | 19 +- ant-core/src/data/client/file.rs | 13 +- ant-core/src/data/client/merkle.rs | 208 ++++++++++-- ant-core/src/data/client/mod.rs | 2 + ant-core/src/data/client/payment.rs | 14 +- ant-core/src/data/client/quote.rs | 508 +++++++++++++++++++++++++--- ant-core/src/data/error.rs | 13 + ant-core/tests/e2e_adr0003.rs | 186 ++++++++++ ant-core/tests/e2e_payment.rs | 5 +- ant-core/tests/e2e_security.rs | 33 +- ant-core/tests/support/mod.rs | 88 ++++- 12 files changed, 994 insertions(+), 97 deletions(-) create mode 100644 ant-core/tests/e2e_adr0003.rs diff --git a/ant-core/examples/bench-quoting.rs b/ant-core/examples/bench-quoting.rs index 54bf27b3..abf14865 100644 --- a/ant-core/examples/bench-quoting.rs +++ b/ant-core/examples/bench-quoting.rs @@ -531,7 +531,7 @@ async fn bench_merkle_once(client: &Client, rep: usize, concurrency: usize) -> R &addrs_clone, |body| match body { ChunkMessageBody::MerkleCandidateQuoteResponse( - MerkleCandidateQuoteResponse::Success { candidate_node }, + MerkleCandidateQuoteResponse::Success { candidate_node, .. }, ) => match rmp_serde::from_slice::( &candidate_node, ) { diff --git a/ant-core/src/data/client/batch.rs b/ant-core/src/data/client/batch.rs index d277f939..f68e1808 100644 --- a/ant-core/src/data/client/batch.rs +++ b/ant-core/src/data/client/batch.rs @@ -53,6 +53,10 @@ pub struct PreparedChunk { pub payment: SingleNodePayment, /// Peer quotes for building `ProofOfPayment`. pub peer_quotes: Vec<(EncodedPeerId, PaymentQuote)>, + /// ADR-0003: the signed commitments the bound quotes shipped, forwarded as + /// sidecars in the PUT bundle so storers cross-check synchronously. Empty + /// when every quote was baseline (no commitment to pin). + pub commitment_sidecars: Vec>, } /// Chunk paid but not yet stored. Produced by [`Client::batch_pay`]. @@ -210,6 +214,9 @@ fn build_paid_chunks( peer_quotes: chunk.peer_quotes, }, tx_hashes, + // ADR-0003: forward the bound quotes' commitments so storers + // cross-check synchronously; stripped before persistence node-side. + commitment_sidecars: chunk.commitment_sidecars, }; let proof_bytes = serialize_single_node_proof(&proof) @@ -272,11 +279,17 @@ impl Client { // Use node-reported prices directly — no contract price fetch needed. let mut peer_quotes = Vec::with_capacity(quotes_with_peers.len()); let mut quotes_for_payment = Vec::with_capacity(quotes_with_peers.len()); + // ADR-0003: forward each bound quote's commitment sidecar (baseline + // quotes ship none); `get_store_quotes` already verified the binding. + let mut commitment_sidecars = Vec::new(); - for (peer_id, _addrs, quote, price) in quotes_with_peers { + for (peer_id, _addrs, quote, price, commitment) in quotes_with_peers { let encoded = peer_id_to_encoded(&peer_id)?; peer_quotes.push((encoded, quote.clone())); quotes_for_payment.push((quote, price)); + if let Some(sidecar) = commitment { + commitment_sidecars.push(sidecar); + } } let payment = SingleNodePayment::from_quotes(quotes_for_payment) @@ -288,6 +301,7 @@ impl Client { quoted_peers, payment, peer_quotes, + commitment_sidecars, })) } @@ -1095,6 +1109,7 @@ mod tests { quoted_peers: Vec::new(), payment: SingleNodePayment { quotes }, peer_quotes: Vec::new(), + commitment_sidecars: Vec::new(), } } @@ -1206,6 +1221,8 @@ mod tests { rewards_address: RewardsAddress::new([1u8; 20]), pub_key: vec![], signature: vec![], + committed_key_count: 0, + commitment_pin: None, }; (EncodedPeerId::from([i as u8; 32]), quote) }) diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index c5c28ccf..84737ee1 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -139,8 +139,15 @@ pub enum FileChunkPeerStatus { /// One entry in the per-chunk quote list returned by /// [`Client::get_store_quotes`]: the responding peer, its addresses, the -/// signed quote it returned, and the payment amount it is demanding. -type QuoteEntry = (PeerId, Vec, PaymentQuote, Amount); +/// signed quote it returned, the payment amount it is demanding, and (ADR-0003) +/// the opaque signed-commitment blob the node shipped with the quote. +type QuoteEntry = ( + PeerId, + Vec, + PaymentQuote, + Amount, + Option>, +); type DownloadBatchEntry = (usize, std::result::Result); @@ -1330,7 +1337,7 @@ impl Client { // Use the median price × 3 (matches SingleNodePayment::from_quotes // which pays 3x the median to incentivize reliable storage). - let mut prices: Vec = quotes.iter().map(|(_, _, _, price)| *price).collect(); + let mut prices: Vec = quotes.iter().map(|(_, _, _, price, _)| *price).collect(); prices.sort(); let median_price = prices .get(prices.len() / 2) diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index d807128b..08700950 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -13,7 +13,13 @@ use ant_protocol::evm::{ Amount, MerklePaymentCandidateNode, MerklePaymentCandidatePool, MerklePaymentProof, MerkleTree, MidpointProof, PoolCommitment, CANDIDATES_PER_POOL, MAX_LEAVES, }; -use ant_protocol::payment::{serialize_merkle_proof, verify_merkle_candidate_signature}; +use ant_protocol::payment::commitment::{ + commitment_hash, verify_commitment_signature, StorageCommitment, MAX_COMMITMENT_KEY_COUNT, + MAX_COMMITMENT_SIDECAR_BYTES, +}; +use ant_protocol::payment::{ + calculate_price, serialize_merkle_proof, verify_merkle_candidate_signature, +}; use ant_protocol::transport::PeerId; use ant_protocol::{ compute_address, send_and_await_chunk_response, ChunkMessage, ChunkMessageBody, @@ -31,6 +37,80 @@ use xor_name::XorName; /// Default threshold: use merkle payments when chunk count >= this value. pub const DEFAULT_MERKLE_THRESHOLD: usize = 64; +/// ADR-0003 resolve-before-pay gate for a merkle candidate — the merkle-path +/// equivalent of the single-node `quote_commitment_binding_is_valid`. Runs the +/// FULL binding check (shape, cap, exact price, and for bound candidates the +/// commitment parse, peer-binding, signature, `hash == pin`, and +/// `count == key_count`) before the candidate is allowed into a pool the client +/// may pay. `peer_id` is derived from the candidate's `pub_key` +/// (`BLAKE3(pub_key)`), matching the storer. +/// +/// Returns `Ok(())` if the binding fully resolves, else `Err(detail)`. +fn merkle_candidate_binding_is_valid( + peer_id: &PeerId, + candidate: &MerklePaymentCandidateNode, + commitment: &Option>, +) -> std::result::Result<(), String> { + let count = candidate.committed_key_count; + let pin = candidate.commitment_pin; + match (count, pin.is_some()) { + (0, false) | (1.., true) => {} + (1.., false) => { + return Err(format!( + "committed_key_count={count} > 0 but commitment_pin is None (unauditable count)" + )); + } + (0, true) => { + return Err("committed_key_count=0 with a commitment_pin (incoherent baseline)".into()); + } + } + if count > MAX_COMMITMENT_KEY_COUNT { + return Err(format!( + "committed_key_count={count} exceeds MAX_COMMITMENT_KEY_COUNT={MAX_COMMITMENT_KEY_COUNT}" + )); + } + let expected = calculate_price(count as usize); + if candidate.price != expected { + return Err(format!( + "price {} does not equal calculate_price(committed_key_count={count}) = {expected}", + candidate.price + )); + } + + let Some(pin) = pin else { + return Ok(()); // baseline candidate pins nothing + }; + let Some(blob) = commitment else { + return Err("bound candidate did not ship its commitment; pin is unresolvable".into()); + }; + if blob.len() > MAX_COMMITMENT_SIDECAR_BYTES { + return Err(format!( + "shipped commitment is {} bytes, exceeds MAX_COMMITMENT_SIDECAR_BYTES={MAX_COMMITMENT_SIDECAR_BYTES}", + blob.len() + )); + } + let commitment: StorageCommitment = rmp_serde::from_slice(blob) + .map_err(|e| format!("shipped commitment did not deserialize: {e}"))?; + if compute_address(&commitment.sender_public_key) != *peer_id.as_bytes() + || commitment.sender_peer_id != *peer_id.as_bytes() + { + return Err("shipped commitment is not bound to the candidate peer".into()); + } + if !verify_commitment_signature(&commitment) { + return Err("shipped commitment has an invalid signature".into()); + } + if commitment_hash(&commitment) != Some(pin) { + return Err("shipped commitment does not hash to the candidate's pin".into()); + } + if commitment.key_count != count { + return Err(format!( + "shipped commitment attests key_count={} but the candidate claims {count}", + commitment.key_count + )); + } + Ok(()) +} + /// Payment mode for uploads. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] @@ -83,6 +163,10 @@ pub struct PreparedMerkleBatch { tree: MerkleTree, /// Internal: chunk addresses in order. addresses: Vec<[u8; 32]>, + /// ADR-0003: validated commitment sidecars keyed by `(peer, pin)`, collected + /// during candidate validation. At proof time the winner pool's candidates + /// forward theirs in `MerklePaymentProof.commitment_sidecars`. + commitment_sidecars: HashMap<(PeerId, [u8; 32]), Vec>, } /// Result of checking a merkle upload batch before payment. @@ -443,8 +527,10 @@ impl Client { midpoint_proofs.len() ); - // 3. Collect candidate pools from the network (all pools in parallel) - let candidate_pools = self + // 3. Collect candidate pools from the network (all pools in parallel). + // Each candidate's ADR-0003 binding is verified during collection and + // its commitment sidecar captured (keyed by peer, pin). + let (candidate_pools, commitment_sidecars) = self .build_candidate_pools( &midpoint_proofs, data_type, @@ -466,6 +552,7 @@ impl Client { candidate_pools, tree, addresses: addresses.to_vec(), + commitment_sidecars, }) } @@ -579,14 +666,17 @@ impl Client { data_type: u32, data_size: u64, merkle_payment_timestamp: u64, - ) -> Result> { + ) -> Result<( + Vec, + HashMap<(PeerId, [u8; 32]), Vec>, + )> { let mut pool_futures = FuturesUnordered::new(); for midpoint_proof in midpoint_proofs { let pool_address = midpoint_proof.address(); let mp = midpoint_proof.clone(); pool_futures.push(async move { - let candidate_nodes = self + let (candidate_nodes, sidecars) = self .get_merkle_candidate_pool( &pool_address.0, data_type, @@ -594,19 +684,28 @@ impl Client { merkle_payment_timestamp, ) .await?; - Ok::<_, Error>(MerklePaymentCandidatePool { - midpoint_proof: mp, - candidate_nodes, - }) + Ok::<_, Error>(( + MerklePaymentCandidatePool { + midpoint_proof: mp, + candidate_nodes, + }, + sidecars, + )) }); } let mut pools = Vec::with_capacity(midpoint_proofs.len()); + // ADR-0003: merged map of every validated candidate's commitment sidecar, + // keyed by (peer, pin). At proof time the winner pool's candidates look + // up their sidecars here to forward in the merkle PUT bundle. + let mut sidecars: HashMap<(PeerId, [u8; 32]), Vec> = HashMap::new(); while let Some(result) = pool_futures.next().await { - pools.push(result?); + let (pool, pool_sidecars) = result?; + pools.push(pool); + sidecars.extend(pool_sidecars); } - Ok(pools) + Ok((pools, sidecars)) } /// Collect `CANDIDATES_PER_POOL` (16) merkle candidate quotes from the network. @@ -617,7 +716,10 @@ impl Client { data_type: u32, data_size: u64, merkle_payment_timestamp: u64, - ) -> Result<[MerklePaymentCandidateNode; CANDIDATES_PER_POOL]> { + ) -> Result<( + [MerklePaymentCandidateNode; CANDIDATES_PER_POOL], + HashMap<(PeerId, [u8; 32]), Vec>, + )> { let node = self.network().node(); let timeout = Duration::from_secs(self.config().quote_timeout_secs); @@ -685,12 +787,15 @@ impl Client { &addrs_clone, |body| match body { ChunkMessageBody::MerkleCandidateQuoteResponse( - MerkleCandidateQuoteResponse::Success { candidate_node }, + MerkleCandidateQuoteResponse::Success { + candidate_node, + commitment, + }, ) => { match rmp_serde::from_slice::( &candidate_node, ) { - Ok(node) => Some(Ok(node)), + Ok(node) => Some(Ok((node, commitment))), Err(e) => Some(Err(Error::Serialization(format!( "Failed to deserialize candidate node from {peer_id_clone}: {e}" )))), @@ -742,19 +847,23 @@ impl Client { impl std::future::Future< Output = ( PeerId, - std::result::Result, + std::result::Result<(MerklePaymentCandidateNode, Option>), Error>, ), >, >, target_address: &[u8; 32], merkle_payment_timestamp: u64, - ) -> Result<[MerklePaymentCandidateNode; CANDIDATES_PER_POOL]> { + ) -> Result<( + [MerklePaymentCandidateNode; CANDIDATES_PER_POOL], + HashMap<(PeerId, [u8; 32]), Vec>, + )> { let mut valid: Vec<(PeerId, MerklePaymentCandidateNode)> = Vec::new(); let mut failures: Vec = Vec::new(); + let mut sidecars: HashMap<(PeerId, [u8; 32]), Vec> = HashMap::new(); while let Some((peer_id, result)) = futures.next().await { match result { - Ok(candidate) => { + Ok((candidate, commitment)) => { if !verify_merkle_candidate_signature(&candidate) { warn!("Invalid ML-DSA-65 signature from merkle candidate {peer_id}"); failures.push(format!("{peer_id}: invalid signature")); @@ -765,7 +874,38 @@ impl Client { failures.push(format!("{peer_id}: timestamp mismatch")); continue; } - valid.push((peer_id, candidate)); + // The candidate's identity is `BLAKE3(candidate.pub_key)` — + // this is what the storer derives (verifier.rs) and what proof + // finalization keys the sidecar by. Require it to equal the + // network responder so a two-identity operator cannot answer + // as B while shipping A's commitment. + let candidate_peer = PeerId::from_bytes(compute_address(&candidate.pub_key)); + if candidate_peer != peer_id { + warn!( + "Dropping merkle candidate {peer_id} — pub_key derives {candidate_peer}, \ + not the responding peer" + ); + failures.push(format!("{peer_id}: candidate pub_key/peer mismatch")); + continue; + } + // ADR-0003: the FULL resolve-before-pay binding check, same as + // the single-node path — a candidate priced off its committed + // count, or shipping an unresolvable/forged commitment, is + // dropped before it can enter a pool the client pays. Checked + // against the CANDIDATE peer (the one the storer audits). + if let Err(detail) = + merkle_candidate_binding_is_valid(&candidate_peer, &candidate, &commitment) + { + warn!("Dropping merkle candidate {peer_id} — ADR-0003 binding invalid: {detail}"); + failures.push(format!("{peer_id}: bad commitment binding ({detail})")); + continue; + } + // Key the sidecar by the CANDIDATE peer (== BLAKE3(pub_key)) so + // proof finalization, which derives the same key, finds it. + if let (Some(pin), Some(blob)) = (candidate.commitment_pin, commitment) { + sidecars.insert((candidate_peer, pin), blob); + } + valid.push((candidate_peer, candidate)); } Err(e) => { debug!("Failed to get merkle candidate from {peer_id}: {e}"); @@ -791,9 +931,11 @@ impl Client { .map(|(_, candidate)| candidate) .collect(); - candidates - .try_into() - .map_err(|_| Error::Payment("Failed to convert candidates to fixed array".to_string())) + let array: [MerklePaymentCandidateNode; CANDIDATES_PER_POOL] = + candidates.try_into().map_err(|_| { + Error::Payment("Failed to convert candidates to fixed array".to_string()) + })?; + Ok((array, sidecars)) } /// Upload chunks using pre-computed merkle proofs from a batch payment. @@ -1304,6 +1446,19 @@ pub fn finalize_merkle_batch( )) })?; + // ADR-0003: collect the winner pool's candidate commitment sidecars (those + // that were bound + validated during collection), to forward in each proof + // so the storer can cross-check synchronously. Built once for the pool. + let winner_sidecars: Vec> = winner_pool + .candidate_nodes + .iter() + .filter_map(|c| { + let pin = c.commitment_pin?; + let peer = PeerId::from_bytes(compute_address(&c.pub_key)); + prepared.commitment_sidecars.get(&(peer, pin)).cloned() + }) + .collect(); + // Generate proofs for each chunk info!("Generating merkle proofs for {chunk_count} chunks"); let mut proofs = HashMap::with_capacity(chunk_count); @@ -1318,7 +1473,9 @@ pub fn finalize_merkle_batch( )) })?; - let merkle_proof = MerklePaymentProof::new(*xorname, address_proof, winner_pool.clone()); + let mut merkle_proof = + MerklePaymentProof::new(*xorname, address_proof, winner_pool.clone()); + merkle_proof.commitment_sidecars = winner_sidecars.clone(); let tagged_bytes = serialize_merkle_proof(&merkle_proof) .map_err(|e| Error::Serialization(format!("Failed to serialize merkle proof: {e}")))?; @@ -1611,6 +1768,8 @@ mod tests { reward_address: RewardsAddress::new([i as u8; 20]), merkle_payment_timestamp: timestamp, signature: vec![i as u8; 64], + committed_key_count: 0, + commitment_pin: None, }); let pool = MerklePaymentCandidatePool { @@ -1649,6 +1808,8 @@ mod tests { reward_address: ant_protocol::evm::RewardsAddress::new([0u8; 20]), merkle_payment_timestamp: 1000, signature: vec![0u8; 64], + committed_key_count: 0, + commitment_pin: None, }; // Timestamp check: 1000 != 2000 @@ -1668,6 +1829,8 @@ mod tests { reward_address: RewardsAddress::new([i as u8; 20]), merkle_payment_timestamp: timestamp, signature: vec![i as u8; 64], + committed_key_count: 0, + commitment_pin: None, }) } @@ -1703,6 +1866,7 @@ mod tests { candidate_pools, tree, addresses: addrs, + commitment_sidecars: HashMap::new(), } } diff --git a/ant-core/src/data/client/mod.rs b/ant-core/src/data/client/mod.rs index ddd085ad..5214df9b 100644 --- a/ant-core/src/data/client/mod.rs +++ b/ant-core/src/data/client/mod.rs @@ -89,6 +89,7 @@ pub(crate) fn classify_error(err: &Error) -> Outcome { | Error::CostEstimationInconclusive(_) | Error::Cancelled(_) | Error::BadQuoteBinding { .. } + | Error::BadQuoteCommitment { .. } // A remote node responded with a structured rejection — the // transport round-trip succeeded, so the node declined at the // application layer (payment/disk/quote/pool). Not a local @@ -770,6 +771,7 @@ mod tests { | Error::Cancelled(_) | Error::PartialUpload { .. } | Error::BadQuoteBinding { .. } + | Error::BadQuoteCommitment { .. } | Error::RemotePut { .. } | Error::CloseGroupShortfall(_) => (), }; diff --git a/ant-core/src/data/client/payment.rs b/ant-core/src/data/client/payment.rs index 8bcad918..3d2fb3e7 100644 --- a/ant-core/src/data/client/payment.rs +++ b/ant-core/src/data/client/payment.rs @@ -68,11 +68,20 @@ impl Client { // Use node-reported prices directly — no contract price fetch needed. let mut peer_quotes = Vec::with_capacity(quotes_with_peers.len()); let mut quotes_for_payment = Vec::with_capacity(quotes_with_peers.len()); - - for (peer_id, _addrs, quote, price) in quotes_with_peers { + // ADR-0003: forward the signed commitment each bound quote shipped, so + // the storers can cross-check the quote's count against the original + // commitment synchronously ("the commitment arrived with the quote"). + // A baseline quote ships none. `get_store_quotes` already verified each + // quote's forced-price binding, so anything here is payable. + let mut commitment_sidecars = Vec::new(); + + for (peer_id, _addrs, quote, price, commitment) in quotes_with_peers { let encoded = peer_id_to_encoded(&peer_id)?; peer_quotes.push((encoded, quote.clone())); quotes_for_payment.push((quote, price)); + if let Some(sidecar) = commitment { + commitment_sidecars.push(sidecar); + } } // 3. Create SingleNodePayment (sorts by price, selects median) @@ -102,6 +111,7 @@ impl Client { let proof = PaymentProof { proof_of_payment: ProofOfPayment { peer_quotes }, tx_hashes, + commitment_sidecars, }; let proof_bytes = serialize_single_node_proof(&proof) diff --git a/ant-core/src/data/client/quote.rs b/ant-core/src/data/client/quote.rs index 68ff69b7..2737f191 100644 --- a/ant-core/src/data/client/quote.rs +++ b/ant-core/src/data/client/quote.rs @@ -8,6 +8,12 @@ use crate::data::client::Client; use crate::data::client::PUT_TARGET_WIDTH; use crate::data::error::{Error, Result}; use ant_protocol::evm::{Amount, PaymentQuote}; +use ant_protocol::payment::calculate_price; +use ant_protocol::payment::commitment::{ + commitment_hash, verify_commitment_signature, StorageCommitment, MAX_COMMITMENT_KEY_COUNT, + MAX_COMMITMENT_SIDECAR_BYTES, +}; +use ant_protocol::payment::{verify_quote_content, verify_quote_signature}; use ant_protocol::transport::{ DHTNode, MultiAddr, P2PNode, PeerId, ResponderView, WitnessedCloseGroup, }; @@ -54,6 +60,21 @@ const QUOTE_COLLECTION_TIMEOUT_SECS: u64 = 120; /// `ant-protocol` re-exports it (`pqc::ops::ML_DSA_65_PUBLIC_KEY_SIZE`). const ML_DSA_PUB_KEY_LEN: usize = 1952; +/// One collected quote: the responding peer, its addresses, the signed quote, +/// the price it demands, and (ADR-0003) the opaque signed-commitment blob the +/// node shipped alongside the quote (`None` for a baseline quote), to be +/// forwarded as a sidecar in the PUT bundle. +type QuotedPeer = ( + PeerId, + Vec, + PaymentQuote, + Amount, + Option>, +); + +/// Per-peer classification outcome: `(quote, price, commitment)` on success. +type QuoteResult = std::result::Result<(PaymentQuote, Amount, Option>), Error>; + /// Check that a quote's `pub_key` is well-formed and BLAKE3-hashes to the /// claimed `peer_id`. /// @@ -64,11 +85,10 @@ const ML_DSA_PUB_KEY_LEN: usize = 1952; /// failing either check causes the storer to reject the entire close-group /// proof and burn the chunk's payment. /// -/// We mirror the cheap structural check here. The storer also runs -/// `verify_quote_content` and `verify_quote_signature`; those are ML-DSA -/// verifications (~1 ms per requested quote) and are deliberately NOT mirrored -/// on the client to keep upload latency unchanged. They are tracked as a -/// follow-up if a real attack surfaces them. +/// This is the cheap structural pre-check. ADR-0003 additionally has the client +/// run `verify_quote_content` + `verify_quote_signature` (the full ML-DSA check) +/// in [`classify_quote_response`] before paying, so a quote the storer would +/// reject never gets paid. fn quote_binding_is_valid(peer_id: &PeerId, quote: &PaymentQuote) -> bool { if quote.pub_key.len() != ML_DSA_PUB_KEY_LEN { return false; @@ -76,19 +96,122 @@ fn quote_binding_is_valid(peer_id: &PeerId, quote: &PaymentQuote) -> bool { compute_address("e.pub_key) == *peer_id.as_bytes() } +/// ADR-0003 client-side resolve-before-pay gate — "the client pays nothing it +/// cannot resolve", the ceiling's load-bearing wall (ADR §"The client pays +/// nothing it cannot resolve"). +/// +/// Runs the **full** binding check before paying, identical to the storer's, +/// using the shared `ant-protocol` commitment type + verifier so client and +/// node can never disagree: +/// 1. **Shape.** `(0, None)` baseline or `(n>0, Some(pin))` bound; the mixed +/// shapes `(n>0, None)` (unauditable count) and `(0, Some)` (incoherent +/// baseline) are rejected. +/// 2. **Cap.** `committed_key_count <= MAX_COMMITMENT_KEY_COUNT` — a count a +/// commitment could never legitimately attest is rejected before pricing. +/// 3. **Forced price.** `price == calculate_price(committed_key_count)`, by +/// exact recomputation with the shared `calculate_price` — never inverted. +/// 4. **Resolution (bound quotes).** The shipped commitment must: parse as a +/// `StorageCommitment`, be bound to the quoting peer +/// (`BLAKE3(sender_public_key) == sender_peer_id`), have a valid ML-DSA-65 +/// signature, hash to the quote's `commitment_pin` +/// (`commitment_hash == pin`), and attest exactly the claimed count +/// (`key_count == committed_key_count`). A withheld, unparseable, wrong-pin, +/// mis-bound, or count-mismatched commitment is unresolvable → the quote is +/// dropped before payment. +/// +/// Returns `Ok(())` if the binding fully resolves, or `Err(detail)` naming the +/// rule that failed. +fn quote_commitment_binding_is_valid( + peer_id: &PeerId, + quote: &PaymentQuote, + commitment: &Option>, +) -> std::result::Result<(), String> { + let count = quote.committed_key_count; + let pin = quote.commitment_pin; + match (count, pin.is_some()) { + (0, false) | (1.., true) => {} + (1.., false) => { + return Err(format!( + "committed_key_count={count} > 0 but commitment_pin is None (unauditable count)" + )); + } + (0, true) => { + return Err("committed_key_count=0 with a commitment_pin (incoherent baseline)".into()); + } + } + if count > MAX_COMMITMENT_KEY_COUNT { + return Err(format!( + "committed_key_count={count} exceeds MAX_COMMITMENT_KEY_COUNT={MAX_COMMITMENT_KEY_COUNT}" + )); + } + // Forced price: exact recomputation, never inversion. + let expected = calculate_price(count as usize); + if quote.price != expected { + return Err(format!( + "price {} does not equal calculate_price(committed_key_count={count}) = {expected}", + quote.price + )); + } + + // Baseline `(0, None)` pins nothing — fully resolved by the checks above. + let Some(pin) = pin else { + return Ok(()); + }; + + // Bound quote: the commitment MUST have arrived and MUST resolve the pin. + let Some(blob) = commitment else { + return Err( + "bound quote did not ship its commitment; the pin is unresolvable so the quote \ + is dropped before payment" + .into(), + ); + }; + // Cap before parsing: bound the deserialize work a malicious responder can + // force, and never forward an oversized blob in the PUT bundle. + if blob.len() > MAX_COMMITMENT_SIDECAR_BYTES { + return Err(format!( + "shipped commitment is {} bytes, exceeds MAX_COMMITMENT_SIDECAR_BYTES={MAX_COMMITMENT_SIDECAR_BYTES}", + blob.len() + )); + } + let commitment: StorageCommitment = rmp_serde::from_slice(blob).map_err(|e| { + format!("shipped commitment did not deserialize as a StorageCommitment: {e}") + })?; + + // Peer binding: the commitment must belong to the quoting peer, exactly as + // the storer derives a candidate's peer id (`BLAKE3(pub_key)`). + if compute_address(&commitment.sender_public_key) != *peer_id.as_bytes() + || commitment.sender_peer_id != *peer_id.as_bytes() + { + return Err("shipped commitment is not bound to the quoting peer".into()); + } + if !verify_commitment_signature(&commitment) { + return Err("shipped commitment has an invalid signature".into()); + } + if commitment_hash(&commitment) != Some(pin) { + return Err("shipped commitment does not hash to the quote's pin".into()); + } + if commitment.key_count != count { + return Err(format!( + "shipped commitment attests key_count={} but the quote claims {count}", + commitment.key_count + )); + } + Ok(()) +} + /// Classification of a `ChunkQuoteResponse::Success` body for a single peer. /// /// Mirrors the storer-side `validate_peer_bindings` check from /// `ant-node/src/payment/verifier.rs` — the cheap BLAKE3 binding — /// so we drop misbehaving peers' quotes before payment. /// -/// We deliberately do NOT mirror the storer's `verify_quote_signature` -/// (ML-DSA-65 verify, ~1 ms × CLOSE_GROUP_SIZE × every chunk) or -/// `verify_quote_content`. Those are useful defense-in-depth for an -/// attacker who self-consistently crafts a signed-but-stolen or wrong- -/// content quote, but they are NOT cheap and are out of scope for this -/// fix. Adding them changes upload latency materially. Track them as a -/// follow-up if a real attack surfaces them. +/// ADR-0003: the client now ALSO runs the storer's `verify_quote_content` and +/// `verify_quote_signature` (ML-DSA-65) before paying, so "the client pays +/// nothing it cannot resolve" covers the quote's own validity too, not just the +/// commitment binding. This matches what the merkle path already does +/// client-side and costs ~1 ms × CLOSE_GROUP_SIZE per chunk — accepted, since +/// paying a quote the storer then rejects burns the on-chain payment. /// /// Pulling the logic out of the async closure lets us unit-test the /// primary defense (not just the post-collect defensive filter). @@ -101,12 +224,21 @@ fn quote_binding_is_valid(peer_id: &PeerId, quote: &PaymentQuote) -> bool { /// - `Err(Error::BadQuoteBinding { .. })` — bad binding (mirrors the /// storer-side rejection). Outer collector counts these via the typed /// variant (no string matching). +/// - `Err(Error::BadQuoteCommitment { .. })` — ADR-0003 forced-price binding +/// failed (price off the curve, incoherent shape, or a bound quote that did +/// not ship its commitment); dropped before payment like a bad binding. /// - `Err(Error::Serialization(...))` — the quote bytes did not deserialize. +/// +/// On success the returned commitment is the opaque signed-commitment blob the +/// node shipped with the quote (`None` for a baseline quote), to be forwarded +/// as a sidecar in the PUT bundle. fn classify_quote_response( peer_id: &PeerId, + expected_content: &[u8; 32], quote_bytes: &[u8], already_stored: bool, -) -> std::result::Result<(PaymentQuote, Amount), Error> { + commitment: Option>, +) -> std::result::Result<(PaymentQuote, Amount, Option>), Error> { let payment_quote = rmp_serde::from_slice::(quote_bytes).map_err(|e| { Error::Serialization(format!("Failed to deserialize quote from {peer_id}: {e}")) })?; @@ -132,22 +264,51 @@ fn classify_quote_response( }); } + // ADR-0003 "the client runs the full binding check": verify the quote's OWN + // ML-DSA-65 signature and that it is for THIS content, before paying — + // exactly what the storer checks and what the merkle path already does + // client-side. A quote with a valid pub_key binding but a bad signature or + // wrong content would otherwise be paid and then rejected by the storer. + if !verify_quote_content(&payment_quote, expected_content) { + return Err(Error::BadQuoteBinding { + peer_id: peer_id.to_string(), + detail: "quote content does not match the requested address".to_string(), + }); + } + if !verify_quote_signature(&payment_quote) { + return Err(Error::BadQuoteBinding { + peer_id: peer_id.to_string(), + detail: "quote ML-DSA-65 signature is invalid".to_string(), + }); + } + + // ADR-0003 forced-price gate: drop a quote whose price is not exactly the + // public formula of its committed count, whose (count, pin) shape is + // incoherent, or which is bound but did not ship its commitment. The storer + // re-runs the arithmetic and would reject the bundle; we drop it here so we + // never pay a quote we cannot resolve. + if let Err(detail) = quote_commitment_binding_is_valid(peer_id, &payment_quote, &commitment) { + warn!("Dropping response from {peer_id} — ADR-0003 binding invalid: {detail}"); + return Err(Error::BadQuoteCommitment { + peer_id: peer_id.to_string(), + detail, + }); + } + if already_stored { debug!("Peer {peer_id} already has chunk"); return Err(Error::AlreadyStored); } let price = payment_quote.price; debug!("Received quote from {peer_id}: price = {price}"); - Ok((payment_quote, price)) + Ok((payment_quote, price, commitment)) } /// Drop quotes whose `pub_key` does not BLAKE3-hash to the peer that supplied /// them. Logs each dropped quote at WARN. -fn drop_quotes_with_bad_bindings( - quotes: &mut Vec<(PeerId, Vec, PaymentQuote, Amount)>, -) -> usize { +fn drop_quotes_with_bad_bindings(quotes: &mut Vec) -> usize { let before = quotes.len(); - quotes.retain(|(peer_id, _, quote, _)| { + quotes.retain(|(peer_id, _, quote, _, _)| { if quote_binding_is_valid(peer_id, quote) { true } else { @@ -1093,7 +1254,7 @@ impl Client { // Check already-stored: only count votes from the closest CLOSE_GROUP_SIZE peers. if !already_stored_peers.is_empty() { let mut all_peers_by_distance: Vec<(bool, [u8; 32])> = Vec::new(); - for (peer_id, _, _, _) in "es { + for (peer_id, _, _, _, _) in "es { all_peers_by_distance.push((false, peer_xor_distance(peer_id, address))); } for (_, dist) in &already_stored_peers { @@ -1186,39 +1347,71 @@ mod tests { struct Keypair { peer_id: PeerId, pub_key_bytes: Vec, + secret_key_bytes: Vec, } fn gen_keypair() -> Keypair { let ml_dsa = MlDsa65::new(); - let (pub_key, _sk) = ml_dsa.generate_keypair().expect("ML-DSA-65 keygen"); + let (pub_key, sk) = ml_dsa.generate_keypair().expect("ML-DSA-65 keygen"); let pub_key_bytes = pub_key.as_bytes().to_vec(); let peer_id = PeerId::from_bytes(compute_address(&pub_key_bytes)); Keypair { peer_id, pub_key_bytes, + secret_key_bytes: sk.as_bytes().to_vec(), } } + /// Build a PROPERLY-SIGNED baseline quote for `content`, signed by a real + /// ML-DSA-65 key whose `BLAKE3(pub_key)` is the returned peer id. Passes the + /// client's full classifier gate (binding + content + signature + price). + fn signed_baseline_quote(content: [u8; 32]) -> (PeerId, PaymentQuote) { + use ant_protocol::pqc::ops::MlDsaSecretKey; + let kp = gen_keypair(); + let mut quote = PaymentQuote { + content: XorName(content), + timestamp: SystemTime::UNIX_EPOCH, + price: calculate_price(0), + rewards_address: RewardsAddress::new([0u8; 20]), + pub_key: kp.pub_key_bytes.clone(), + signature: Vec::new(), + committed_key_count: 0, + commitment_pin: None, + }; + let ml_dsa = MlDsa65::new(); + let sk = MlDsaSecretKey::from_bytes(&kp.secret_key_bytes).expect("sk"); + let msg = quote.bytes_for_sig(); + quote.signature = ml_dsa.sign(&sk, &msg).expect("sign").as_bytes().to_vec(); + (kp.peer_id, quote) + } + /// Build a quote tuple whose `pub_key` correctly hashes to its peer_id. /// Signature is left empty: this filter does not verify signatures. - fn good_quote_real() -> (PeerId, Vec, PaymentQuote, Amount) { + /// + /// The quote is a valid ADR-0003 **baseline**: `(0, None)` priced at + /// `calculate_price(0)`, so it passes the forced-price gate in + /// `classify_quote_response`. The 5th tuple element is the (absent) + /// commitment sidecar. + fn good_quote_real() -> QuotedPeer { let kp = gen_keypair(); let quote = PaymentQuote { content: XorName([0u8; 32]), timestamp: SystemTime::UNIX_EPOCH, - price: Amount::ZERO, + price: calculate_price(0), rewards_address: RewardsAddress::new([0u8; 20]), pub_key: kp.pub_key_bytes, signature: Vec::new(), + committed_key_count: 0, + commitment_pin: None, }; - (kp.peer_id, Vec::new(), quote, Amount::ZERO) + (kp.peer_id, Vec::new(), quote, calculate_price(0), None) } /// Build a quote tuple where the quote carries a different keypair's /// `pub_key` than the peer_id derives from. Mirrors the production /// failure shape: peer A advertised on the transport, but the quote /// carries peer B's key. - fn bad_quote_real() -> (PeerId, Vec, PaymentQuote, Amount) { + fn bad_quote_real() -> QuotedPeer { let claimed = gen_keypair(); let signing = gen_keypair(); assert_ne!(claimed.pub_key_bytes, signing.pub_key_bytes); @@ -1226,12 +1419,14 @@ mod tests { let quote = PaymentQuote { content: XorName([0u8; 32]), timestamp: SystemTime::UNIX_EPOCH, - price: Amount::ZERO, + price: calculate_price(0), rewards_address: RewardsAddress::new([0u8; 20]), pub_key: signing.pub_key_bytes, signature: Vec::new(), + committed_key_count: 0, + commitment_pin: None, }; - (claimed.peer_id, Vec::new(), quote, Amount::ZERO) + (claimed.peer_id, Vec::new(), quote, calculate_price(0), None) } fn witnessed_test_node(seed: u8) -> DHTNode { @@ -1319,7 +1514,7 @@ mod tests { #[test] fn binding_accepts_real_self_consistent_keypair() { - let (peer_id, _, quote, _) = good_quote_real(); + let (peer_id, _, quote, _, _) = good_quote_real(); // Property under test: the predicate accepts a quote whose pub_key // genuinely belongs to the claimed peer. assert!(quote_binding_is_valid(&peer_id, "e)); @@ -1329,7 +1524,7 @@ mod tests { #[test] fn binding_rejects_real_crossed_keypair() { - let (peer_id, _, quote, _) = bad_quote_real(); + let (peer_id, _, quote, _, _) = bad_quote_real(); assert!(!quote_binding_is_valid(&peer_id, "e)); assert!(!storer_binding_would_accept(&peer_id, "e)); } @@ -1348,6 +1543,8 @@ mod tests { rewards_address: RewardsAddress::new([0u8; 20]), pub_key: oversized, signature: Vec::new(), + committed_key_count: 0, + commitment_pin: None, }; // BLAKE3(pub_key) DOES equal the peer_id we constructed, so the // bare hash check would pass — but the length guard must reject. @@ -1370,6 +1567,8 @@ mod tests { rewards_address: RewardsAddress::new([0u8; 20]), pub_key: undersized, signature: Vec::new(), + committed_key_count: 0, + commitment_pin: None, }; assert!(!quote_binding_is_valid(&peer_id, "e)); assert!(!storer_binding_would_accept(&peer_id, "e)); @@ -1854,7 +2053,7 @@ mod tests { // the binding, so this asserts the binding-only filter is correct // for binding-only failures (other failure modes are filtered by // the per-peer classifier upstream). - for (peer_id, _, quote, _) in "es { + for (peer_id, _, quote, _, _) in "es { assert!( storer_binding_would_accept(peer_id, quote), "every retained quote must satisfy the full storer-side spec" @@ -1869,7 +2068,7 @@ mod tests { let dropped = drop_quotes_with_bad_bindings(&mut quotes); assert_eq!(dropped, 0); assert_eq!(quotes.len(), before); - for (peer_id, _, quote, _) in "es { + for (peer_id, _, quote, _, _) in "es { assert!(storer_binding_would_accept(peer_id, quote)); } } @@ -1894,11 +2093,17 @@ mod tests { // signature, content, timestamp, price, rewards_address. The patch // is a filter, not a transformation; this test catches any future // regression that mutates a retained quote. - let (peer_id, addrs, original_quote, amount) = good_quote_real(); - let mut quotes = vec![(peer_id, addrs.clone(), original_quote.clone(), amount)]; + let (peer_id, addrs, original_quote, amount, commitment) = good_quote_real(); + let mut quotes = vec![( + peer_id, + addrs.clone(), + original_quote.clone(), + amount, + commitment, + )]; let _ = drop_quotes_with_bad_bindings(&mut quotes); - let (kept_peer, kept_addrs, kept_quote, kept_amount) = + let (kept_peer, kept_addrs, kept_quote, kept_amount, _kept_commitment) = quotes.pop().expect("the good quote must survive filtering"); assert_eq!(kept_peer.as_bytes(), peer_id.as_bytes()); assert_eq!(kept_addrs.len(), addrs.len()); @@ -1955,7 +2160,7 @@ mod tests { // Step 1: prove the storer would reject the pre-filter set. let storer_would_reject_count = quotes .iter() - .filter(|(p, _, q, _)| !storer_binding_would_accept(p, q)) + .filter(|(p, _, q, _, _)| !storer_binding_would_accept(p, q)) .count(); assert_eq!( storer_would_reject_count, 1, @@ -1967,7 +2172,7 @@ mod tests { assert_eq!(dropped, 1, "exactly the crossed-key quote must be filtered"); // Step 3: prove the storer would accept every survivor under the FULL spec. - for (peer_id, _, quote, _) in "es { + for (peer_id, _, quote, _, _) in "es { assert!( storer_binding_would_accept(peer_id, quote), "every post-filter quote must be accepted by the storer spec — \ @@ -2003,7 +2208,7 @@ mod tests { "this is the precondition for InsufficientPeers downstream" ); // Sanity: every survivor is storer-acceptable under the full spec. - for (peer_id, _, quote, _) in "es { + for (peer_id, _, quote, _, _) in "es { assert!(storer_binding_would_accept(peer_id, quote)); } } @@ -2028,23 +2233,54 @@ mod tests { #[test] fn classifier_accepts_real_self_consistent_quote() { - let (peer_id, _, quote, _) = good_quote_real(); + // A properly-signed baseline quote for the requested content passes the + // full client gate (binding + content + signature + price). + let content = [7u8; 32]; + let (peer_id, quote) = signed_baseline_quote(content); let bytes = serialize_quote("e); - let result = classify_quote_response(&peer_id, &bytes, false); + let result = classify_quote_response(&peer_id, &content, &bytes, false, None); match result { - Ok((q, price)) => { + Ok((q, price, commitment)) => { assert_eq!(q.pub_key, quote.pub_key); assert_eq!(price, quote.price); + assert!(commitment.is_none(), "baseline quote ships no commitment"); } Err(e) => panic!("expected Ok, got {e}"), } } + #[test] + fn classifier_rejects_quote_with_invalid_signature() { + // A quote whose pub_key binds correctly but whose signature is bogus is + // dropped BEFORE payment (the storer would reject it and burn the pay). + let content = [7u8; 32]; + let (peer_id, mut quote) = signed_baseline_quote(content); + quote.signature = vec![0u8; quote.signature.len()]; // corrupt the signature + let bytes = serialize_quote("e); + let result = classify_quote_response(&peer_id, &content, &bytes, false, None); + assert!( + matches!(result, Err(Error::BadQuoteBinding { .. })), + "a quote with an invalid signature must be rejected; got {result:?}" + ); + } + + #[test] + fn classifier_rejects_quote_for_wrong_content() { + // A validly-signed quote for a DIFFERENT address is dropped before pay. + let (peer_id, quote) = signed_baseline_quote([7u8; 32]); + let bytes = serialize_quote("e); + let result = classify_quote_response(&peer_id, &[9u8; 32], &bytes, false, None); + assert!( + matches!(result, Err(Error::BadQuoteBinding { .. })), + "a quote for the wrong content must be rejected; got {result:?}" + ); + } + #[test] fn classifier_rejects_crossed_keypair_with_typed_error() { - let (peer_id, _, quote, _) = bad_quote_real(); + let (peer_id, _, quote, _, _) = bad_quote_real(); let bytes = serialize_quote("e); - let result = classify_quote_response(&peer_id, &bytes, false); + let result = classify_quote_response(&peer_id, &[0u8; 32], &bytes, false, None); match result { Err(Error::BadQuoteBinding { peer_id: pid, @@ -2072,10 +2308,10 @@ mod tests { /// unfiltered." #[test] fn classifier_rejects_already_stored_vote_from_bad_binding_peer() { - let (peer_id, _, quote, _) = bad_quote_real(); + let (peer_id, _, quote, _, _) = bad_quote_real(); let bytes = serialize_quote("e); // The peer claims already_stored=true, but its quote has a crossed key. - let result = classify_quote_response(&peer_id, &bytes, true); + let result = classify_quote_response(&peer_id, &[0u8; 32], &bytes, true, None); assert!( matches!(result, Err(Error::BadQuoteBinding { .. })), "crossed-key peer must be classified BadQuoteBinding even when \ @@ -2087,9 +2323,10 @@ mod tests { /// passing the bind-check). This is the contrast to the test above. #[test] fn classifier_honours_already_stored_vote_from_good_binding_peer() { - let (peer_id, _, quote, _) = good_quote_real(); + let content = [7u8; 32]; + let (peer_id, quote) = signed_baseline_quote(content); let bytes = serialize_quote("e); - let result = classify_quote_response(&peer_id, &bytes, true); + let result = classify_quote_response(&peer_id, &content, &bytes, true, None); assert!( matches!(result, Err(Error::AlreadyStored)), "honest peer's already_stored vote must be honoured; got {result:?}" @@ -2098,9 +2335,9 @@ mod tests { #[test] fn classifier_returns_serialization_error_on_bad_bytes() { - let (peer_id, _, _, _) = good_quote_real(); + let (peer_id, _, _, _, _) = good_quote_real(); let garbage = b"this is not a valid msgpack PaymentQuote".to_vec(); - let result = classify_quote_response(&peer_id, &garbage, false); + let result = classify_quote_response(&peer_id, &[0u8; 32], &garbage, false, None); assert!( matches!(result, Err(Error::Serialization(_))), "garbage bytes must produce a Serialization error; got {result:?}" @@ -2111,21 +2348,19 @@ mod tests { /// independent storer-spec re-derivation across mixed responders. #[test] fn classifier_verdict_matches_storer_binding_spec_for_mixed_responders() { - let mut responders: Vec<(PeerId, PaymentQuote)> = (0..12) - .map(|_| { - let (p, _, q, _) = good_quote_real(); - (p, q) - }) - .collect(); + let content = [7u8; 32]; + let mut responders: Vec<(PeerId, PaymentQuote)> = + (0..12).map(|_| signed_baseline_quote(content)).collect(); for _ in 0..4 { - let (p, _, q, _) = bad_quote_real(); + let (p, _, q, _, _) = bad_quote_real(); responders.push((p, q)); } for (peer_id, quote) in &responders { let bytes = serialize_quote(quote); let storer_verdict = storer_binding_would_accept(peer_id, quote); - let classifier_verdict = classify_quote_response(peer_id, &bytes, false).is_ok(); + let classifier_verdict = + classify_quote_response(peer_id, &content, &bytes, false, None).is_ok(); assert_eq!( classifier_verdict, storer_verdict, "classifier and storer-binding-spec must agree on every responder \ @@ -2134,4 +2369,167 @@ mod tests { ); } } + + // ============================================================ + // ADR-0003: quote_commitment_binding_is_valid (forced-price gate) + // + // Mirrors the storer-side `binding_violation` in + // `ant-node/src/payment/verifier.rs`. The client runs this before + // paying so it never pays a quote the storer's arithmetic gate would + // reject. The client now runs the FULL check (shape, cap, exact price, + // and for bound quotes: parse + peer-binding + signature + hash==pin + + // count==key_count) using the shared ant-protocol commitment type, so an + // unresolvable/forged commitment is never paid. A live resolve against a + // REAL signed commitment is proven in the e2e suite (e2e_adr0003.rs). + // ============================================================ + + /// A throwaway peer id for tests that fail BEFORE commitment resolution + /// (shape/cap/price checks don't depend on the peer). + fn any_peer() -> PeerId { + PeerId::from_bytes([0u8; 32]) + } + + /// Build a quote carrying a specific `(count, pin, price)` binding. + fn quote_with_binding( + committed_key_count: u32, + commitment_pin: Option<[u8; 32]>, + price: Amount, + ) -> PaymentQuote { + PaymentQuote { + content: XorName([0u8; 32]), + timestamp: SystemTime::UNIX_EPOCH, + price, + rewards_address: RewardsAddress::new([0u8; 20]), + pub_key: Vec::new(), + signature: Vec::new(), + committed_key_count, + commitment_pin, + } + } + + #[test] + fn binding_baseline_ok_only_at_baseline_price() { + // (0, None) priced at calculate_price(0) is the valid baseline. + let q = quote_with_binding(0, None, calculate_price(0)); + assert!(quote_commitment_binding_is_valid(&any_peer(), &q, &None).is_ok()); + + // (0, None) priced above baseline is rejected — the forged-shape + // bypass (strip the pin, charge more than the empty-node price). + let q = quote_with_binding(0, None, calculate_price(500)); + assert!(quote_commitment_binding_is_valid(&any_peer(), &q, &None).is_err()); + } + + #[test] + fn binding_rejects_incoherent_shapes() { + // count > 0 but no pin: unauditable. + let q = quote_with_binding(500, None, calculate_price(500)); + assert!(quote_commitment_binding_is_valid(&any_peer(), &q, &None).is_err()); + // count 0 but a pin: incoherent baseline. + let q = quote_with_binding(0, Some([9u8; 32]), calculate_price(0)); + assert!(quote_commitment_binding_is_valid(&any_peer(), &q, &None).is_err()); + } + + #[test] + fn binding_rejects_count_above_cap() { + let over = MAX_COMMITMENT_KEY_COUNT + 1; + let q = quote_with_binding(over, Some([9u8; 32]), calculate_price(over as usize)); + assert!( + quote_commitment_binding_is_valid(&any_peer(), &q, &Some(vec![1u8; 16])).is_err(), + "a count above MAX_COMMITMENT_KEY_COUNT must be rejected before payment" + ); + } + + #[test] + fn binding_rejects_on_curve_wrong_count() { + // Priced for 499 but claims count 500 — on a real price curve but the + // wrong count. Rejected at the exact-price check, before resolution. + let q = quote_with_binding(500, Some([9u8; 32]), calculate_price(499)); + assert!(quote_commitment_binding_is_valid(&any_peer(), &q, &Some(vec![1u8; 16])).is_err()); + } + + #[test] + fn binding_rejects_bound_quote_without_shipped_commitment() { + // A bound quote whose commitment did not arrive is unresolvable, so it + // is dropped before payment even though its price is on the curve. + let q = quote_with_binding(500, Some([9u8; 32]), calculate_price(500)); + assert!( + quote_commitment_binding_is_valid(&any_peer(), &q, &None).is_err(), + "a bound quote missing its commitment must be rejected" + ); + } + + #[test] + fn binding_rejects_garbage_and_wrong_pin_commitment() { + // A bound quote whose shipped commitment is garbage (doesn't even + // deserialize) is rejected — the client never pays an unresolvable pin. + let q = quote_with_binding(500, Some([9u8; 32]), calculate_price(500)); + assert!( + quote_commitment_binding_is_valid(&any_peer(), &q, &Some(vec![0xFF; 8])).is_err(), + "an unparseable commitment must be rejected before payment" + ); + + // A well-formed-but-wrong commitment (valid StorageCommitment bytes that + // do NOT hash to the quote's pin / aren't bound to the peer) is also + // rejected. We serialize a real StorageCommitment shape with mismatched + // fields; it fails peer-binding first, then would fail hash==pin. + let bogus = StorageCommitment { + root: [1u8; 32], + key_count: 500, + sender_peer_id: [2u8; 32], // not the quoting peer + sender_public_key: vec![3u8; 1952], + signature: vec![4u8; 3293], + }; + let blob = rmp_serde::to_vec(&bogus).expect("serialize bogus commitment"); + assert!( + quote_commitment_binding_is_valid(&any_peer(), &q, &Some(blob)).is_err(), + "a commitment not bound to the quoting peer must be rejected before payment" + ); + } + + #[test] + fn binding_rejects_oversized_commitment_before_parsing() { + // A bound quote shipping a blob larger than the sidecar cap is rejected + // before any deserialize attempt (DoS guard on the hot path). + let q = quote_with_binding(500, Some([9u8; 32]), calculate_price(500)); + let huge = Some(vec![0u8; MAX_COMMITMENT_SIDECAR_BYTES + 1]); + assert!( + quote_commitment_binding_is_valid(&any_peer(), &q, &huge).is_err(), + "an oversized commitment blob must be rejected before payment" + ); + } + + #[test] + fn classifier_drops_off_curve_quote_with_typed_error() { + // End-to-end through the classifier: a VALIDLY-SIGNED, correctly-bound + // quote for the right content, but with an off-curve price, is dropped + // as BadQuoteCommitment (the forced-price extraction guard fires after + // the quote's own signature/content checks pass). + use ant_protocol::pqc::ops::MlDsaSecretKey; + let content = [7u8; 32]; + let kp = gen_keypair(); + let mut quote = PaymentQuote { + content: XorName(content), + timestamp: SystemTime::UNIX_EPOCH, + // claims baseline shape but charges a non-baseline price + price: calculate_price(500), + rewards_address: RewardsAddress::new([0u8; 20]), + pub_key: kp.pub_key_bytes.clone(), + signature: Vec::new(), + committed_key_count: 0, + commitment_pin: None, + }; + let ml_dsa = MlDsa65::new(); + let sk = MlDsaSecretKey::from_bytes(&kp.secret_key_bytes).expect("sk"); + quote.signature = ml_dsa + .sign(&sk, "e.bytes_for_sig()) + .expect("sign") + .as_bytes() + .to_vec(); + let bytes = serialize_quote("e); + let result = classify_quote_response(&kp.peer_id, &content, &bytes, false, None); + assert!( + matches!(result, Err(Error::BadQuoteCommitment { .. })), + "off-curve quote must be dropped as BadQuoteCommitment; got {result:?}" + ); + } } diff --git a/ant-core/src/data/error.rs b/ant-core/src/data/error.rs index f6e94386..8f7ed960 100644 --- a/ant-core/src/data/error.rs +++ b/ant-core/src/data/error.rs @@ -119,6 +119,19 @@ pub enum Error { detail: String, }, + /// ADR-0003: a quote's commitment binding does not hold — its price is not + /// `calculate_price(committed_key_count)`, its `(count, pin)` shape is + /// incoherent, or a shipped commitment does not match the pinned count/hash. + /// The storer's arithmetic gate would reject such a quote, so the client + /// drops it before paying ("the client pays nothing it cannot resolve"). + #[error("bad quote commitment from peer {peer_id}: {detail}")] + BadQuoteCommitment { + /// The peer ID we got the quote from. + peer_id: String, + /// Diagnostic detail (which binding rule failed). + detail: String, + }, + /// Not enough disk space for the operation. #[error("insufficient disk space: {0}")] InsufficientDiskSpace(String), diff --git a/ant-core/tests/e2e_adr0003.rs b/ant-core/tests/e2e_adr0003.rs new file mode 100644 index 00000000..448047c4 --- /dev/null +++ b/ant-core/tests/e2e_adr0003.rs @@ -0,0 +1,186 @@ +//! ADR-0003 end-to-end: commitment-bound quote pricing. +//! +//! Proves, against a real in-process QUIC testnet + Anvil EVM, that: +//! +//! 1. A node carrying a live storage commitment emits a COMMITMENT-BOUND quote: +//! `committed_key_count == N`, `commitment_pin == Some`, the price is exactly +//! `calculate_price(N)`, and the signed commitment is SHIPPED in the quote +//! response (ADR-0003 "the commitment arrived with the quote"). +//! 2. The client's forced-price gate ACCEPTS those bound quotes (they are +//! self-consistent) and a full pay → store → retrieve round-trip succeeds — +//! so the storer accepts the bound quotes and the forwarded sidecars too. +//! 3. A node with NO commitment emits a valid BASELINE quote `(0, None)` priced +//! at `calculate_price(0)`, and the full flow still works. +//! +//! The OFF-CURVE rejection (a node charging above its committed count is dropped +//! before payment) is proven directly on the gate in the `quote.rs` unit tests +//! (`classifier_drops_off_curve_quote_with_typed_error` + the +//! `quote_commitment_binding_is_valid` suite) — honest nodes never emit an +//! off-curve quote, so it cannot be surfaced through a live happy-path network. + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +mod support; + +use ant_core::data::{compute_address, Client}; +use ant_protocol::payment::calculate_price; +use ant_protocol::payment::commitment::{ + commitment_hash, verify_commitment_signature, StorageCommitment, +}; +use bytes::Bytes; +use serial_test::serial; +use std::sync::Arc; +use support::{test_client_config, MiniTestnet, DEFAULT_NODE_COUNT}; + +/// The committed key count every node attests in these tests. Picked above the +/// pricing baseline so `calculate_price(N)` is strictly greater than the +/// empty-node baseline — a bound quote is visibly distinct from a baseline one. +const COMMITTED_KEYS: u32 = 9_000; + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr0003_bound_quotes_are_shipped_priced_and_resolve() { + let testnet = MiniTestnet::start_with_commitments(DEFAULT_NODE_COUNT, COMMITTED_KEYS).await; + let node = testnet.node(3).expect("node 3 exists"); + let client = Client::from_node(Arc::clone(&node), test_client_config()) + .with_wallet(testnet.wallet().clone()); + + let content = Bytes::from("adr-0003 bound-quote payload"); + let address = compute_address(&content); + + // Collect quotes directly so we can inspect the ADR-0003 binding the client + // verified before it would pay. + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("quote collection should reach quorum"); + + assert!( + quotes.len() >= ant_protocol::CLOSE_GROUP_SIZE, + "must collect a full close group of bound quotes, got {}", + quotes.len() + ); + + let expected_price = calculate_price(COMMITTED_KEYS as usize); + for (peer_id, _addrs, quote, price, commitment) in "es { + // Bound shape: the count is what the node committed, with a pin. + assert_eq!( + quote.committed_key_count, COMMITTED_KEYS, + "quote from {peer_id} must carry the committed key count" + ); + let pin = quote + .commitment_pin + .expect("a bound quote must carry a commitment pin"); + + // Forced price: exactly calculate_price(N), by recomputation. This is + // the ceiling — a node cannot charge above what it can prove it stores. + assert_eq!( + quote.price, expected_price, + "quote from {peer_id} must be priced at calculate_price(committed_key_count)" + ); + assert_eq!(*price, expected_price); + + // The commitment arrived WITH the quote (no separate fetch needed). + let sidecar = commitment + .as_ref() + .expect("a bound quote response must ship its signed commitment"); + + // FULLY resolve the shipped commitment exactly as the client gate does + // before paying — this is the real anti-cheat invariant, not just + // "non-empty bytes". The commitment must deserialize, carry a valid + // signature, be bound to the quoting peer, hash to the quote's pin, and + // attest exactly the claimed count. + let resolved: StorageCommitment = + rmp_serde::from_slice(sidecar).expect("shipped commitment must deserialize"); + assert!( + verify_commitment_signature(&resolved), + "shipped commitment from {peer_id} must be validly signed" + ); + assert_eq!( + resolved.sender_peer_id, + *peer_id.as_bytes(), + "shipped commitment must be bound to the quoting peer" + ); + assert_eq!( + compute_address(&resolved.sender_public_key), + *peer_id.as_bytes(), + "BLAKE3(sender_public_key) must equal the quoting peer (full peer binding)" + ); + assert_eq!( + commitment_hash(&resolved), + Some(pin), + "shipped commitment must hash to the quote's pin" + ); + assert_eq!( + resolved.key_count, COMMITTED_KEYS, + "shipped commitment must attest exactly the committed key count" + ); + } + + // Full round-trip: the client paid (accepting the bound quotes + forwarding + // the sidecars), the storers accepted the bundle, and the chunk is back. + let stored = client + .chunk_put(content.clone()) + .await + .expect("paid put with commitment-bound quotes must succeed"); + assert_eq!(stored, address); + + let retrieved = client + .chunk_get(&address) + .await + .expect("chunk_get should succeed") + .expect("chunk must be found after storing"); + assert_eq!(retrieved.content.as_ref(), content.as_ref()); + + drop(client); + testnet.teardown().await; +} + +/// The other half: a node with NO commitment emits a valid BASELINE quote +/// `(0, None)` priced at `calculate_price(0)`, and the full flow still works. +/// This guards the baseline branch of the same forced-price gate. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr0003_baseline_quotes_still_work() { + let testnet = MiniTestnet::start(DEFAULT_NODE_COUNT).await; + let node = testnet.node(3).expect("node 3 exists"); + let client = Client::from_node(Arc::clone(&node), test_client_config()) + .with_wallet(testnet.wallet().clone()); + + let content = Bytes::from("adr-0003 baseline payload"); + let address = compute_address(&content); + + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("quote collection should reach quorum"); + + let baseline = calculate_price(0); + for (peer_id, _addrs, quote, _price, commitment) in "es { + assert_eq!( + quote.committed_key_count, 0, + "no-commitment node must quote count 0 ({peer_id})" + ); + assert!( + quote.commitment_pin.is_none(), + "baseline quote pins nothing ({peer_id})" + ); + assert_eq!( + quote.price, baseline, + "baseline quote must price at calculate_price(0) ({peer_id})" + ); + assert!( + commitment.is_none(), + "baseline quote ships no commitment ({peer_id})" + ); + } + + let stored = client + .chunk_put(content.clone()) + .await + .expect("paid put with baseline quotes must succeed"); + assert_eq!(stored, address); + + drop(client); + testnet.teardown().await; +} diff --git a/ant-core/tests/e2e_payment.rs b/ant-core/tests/e2e_payment.rs index 3275c348..311b608a 100644 --- a/ant-core/tests/e2e_payment.rs +++ b/ant-core/tests/e2e_payment.rs @@ -262,7 +262,7 @@ async fn test_quote_collection() { ); // All prices should be > 0 - for (peer_id, _addrs, _quote, price) in "es { + for (peer_id, _addrs, _quote, price, _commitment) in "es { assert!( !price.is_zero(), "Quote price from peer {peer_id} should be > 0" @@ -270,7 +270,8 @@ async fn test_quote_collection() { } // All peer IDs should be unique - let unique_peers: std::collections::HashSet<_> = quotes.iter().map(|(p, _, _, _)| *p).collect(); + let unique_peers: std::collections::HashSet<_> = + quotes.iter().map(|(p, _, _, _, _)| *p).collect(); assert_eq!( unique_peers.len(), quotes.len(), diff --git a/ant-core/tests/e2e_security.rs b/ant-core/tests/e2e_security.rs index 888b418c..5b91ed4f 100644 --- a/ant-core/tests/e2e_security.rs +++ b/ant-core/tests/e2e_security.rs @@ -42,10 +42,14 @@ async fn collect_and_pay(client: &Client, content: &Bytes) -> (PaymentProof, Vec // Build peer_quotes and payment let mut peer_quotes = Vec::with_capacity(quotes.len()); let mut quotes_for_payment = Vec::with_capacity(quotes.len()); - for (peer_id, _addrs, quote, price) in quotes { + let mut commitment_sidecars = Vec::new(); + for (peer_id, _addrs, quote, price, commitment) in quotes { let encoded = EncodedPeerId::new(*peer_id.as_bytes()); peer_quotes.push((encoded, quote.clone())); quotes_for_payment.push((quote, price)); + if let Some(sidecar) = commitment { + commitment_sidecars.push(sidecar); + } } // Pay on-chain @@ -57,6 +61,7 @@ async fn collect_and_pay(client: &Client, content: &Bytes) -> (PaymentProof, Vec let proof = PaymentProof { proof_of_payment: ProofOfPayment { peer_quotes }, tx_hashes, + commitment_sidecars, }; let proof_bytes = serialize_single_node_proof(&proof).expect("serialize proof"); @@ -86,6 +91,7 @@ async fn test_attack_forged_signature() { peer_quotes: tampered_quotes, }, tx_hashes: proof.tx_hashes.clone(), + commitment_sidecars: proof.commitment_sidecars.clone(), }; let tampered_bytes = rmp_serde::to_vec(&tampered_proof).expect("serialize tampered proof"); @@ -192,7 +198,7 @@ async fn test_attack_zero_amount_payment() { let target_peer = quotes.first().expect("should have quotes").0; let mut peer_quotes = Vec::with_capacity(quotes.len()); - for (peer_id, _addrs, quote, _price) in quotes { + for (peer_id, _addrs, quote, _price, _commitment) in quotes { let encoded = EncodedPeerId::new(*peer_id.as_bytes()); peer_quotes.push((encoded, quote)); } @@ -201,6 +207,7 @@ async fn test_attack_zero_amount_payment() { let fake_proof = PaymentProof { proof_of_payment: ProofOfPayment { peer_quotes }, tx_hashes: vec![], + commitment_sidecars: vec![], }; let fake_bytes = rmp_serde::to_vec(&fake_proof).expect("serialize fake proof"); @@ -236,7 +243,7 @@ async fn test_attack_fabricated_tx_hash() { let target_peer = quotes.first().expect("should have quotes").0; let mut peer_quotes = Vec::with_capacity(quotes.len()); - for (peer_id, _addrs, quote, _price) in quotes { + for (peer_id, _addrs, quote, _price, _commitment) in quotes { let encoded = EncodedPeerId::new(*peer_id.as_bytes()); peer_quotes.push((encoded, quote)); } @@ -246,6 +253,7 @@ async fn test_attack_fabricated_tx_hash() { let fake_proof = PaymentProof { proof_of_payment: ProofOfPayment { peer_quotes }, tx_hashes: vec![fake_tx_hash], + commitment_sidecars: vec![], }; let fake_bytes = rmp_serde::to_vec(&fake_proof).expect("serialize fake proof"); @@ -317,6 +325,7 @@ async fn test_attack_corrupted_public_key() { peer_quotes: tampered_quotes, }, tx_hashes: proof.tx_hashes.clone(), + commitment_sidecars: proof.commitment_sidecars.clone(), }; let tampered_bytes = rmp_serde::to_vec(&tampered_proof).expect("serialize tampered proof"); @@ -385,16 +394,20 @@ async fn test_attack_underpayment_single_node() { // to target the median peer after from_quotes() sorts internally. let peer_by_rewards: Vec<(PeerId, RewardsAddress)> = quotes .iter() - .map(|(pid, _, q, _)| (*pid, q.rewards_address)) + .map(|(pid, _, q, _, _)| (*pid, q.rewards_address)) .collect(); // 2. Build SingleNodePayment normally (sorts by price, median gets 3×) let mut peer_quotes = Vec::with_capacity(quotes.len()); let mut quotes_for_payment = Vec::with_capacity(quotes.len()); - for (peer_id, _addrs, quote, price) in quotes { + let mut commitment_sidecars = Vec::new(); + for (peer_id, _addrs, quote, price, commitment) in quotes { let encoded = EncodedPeerId::new(*peer_id.as_bytes()); peer_quotes.push((encoded, quote.clone())); quotes_for_payment.push((quote, price)); + if let Some(sidecar) = commitment { + commitment_sidecars.push(sidecar); + } } let mut payment = SingleNodePayment::from_quotes(quotes_for_payment) @@ -434,6 +447,7 @@ async fn test_attack_underpayment_single_node() { let proof = PaymentProof { proof_of_payment: ProofOfPayment { peer_quotes }, tx_hashes, + commitment_sidecars, }; let proof_bytes = serialize_single_node_proof(&proof).expect("serialize proof"); @@ -473,15 +487,19 @@ async fn test_attack_underpayment_half_price() { let peer_by_rewards: Vec<(PeerId, RewardsAddress)> = quotes .iter() - .map(|(pid, _, q, _)| (*pid, q.rewards_address)) + .map(|(pid, _, q, _, _)| (*pid, q.rewards_address)) .collect(); let mut peer_quotes = Vec::with_capacity(quotes.len()); let mut quotes_for_payment = Vec::with_capacity(quotes.len()); - for (peer_id, _addrs, quote, price) in quotes { + let mut commitment_sidecars = Vec::new(); + for (peer_id, _addrs, quote, price, commitment) in quotes { let encoded = EncodedPeerId::new(*peer_id.as_bytes()); peer_quotes.push((encoded, quote.clone())); quotes_for_payment.push((quote, price)); + if let Some(sidecar) = commitment { + commitment_sidecars.push(sidecar); + } } let mut payment = SingleNodePayment::from_quotes(quotes_for_payment) @@ -512,6 +530,7 @@ async fn test_attack_underpayment_half_price() { let proof = PaymentProof { proof_of_payment: ProofOfPayment { peer_quotes }, tx_hashes, + commitment_sidecars, }; let proof_bytes = serialize_single_node_proof(&proof).expect("serialize proof"); diff --git a/ant-core/tests/support/mod.rs b/ant-core/tests/support/mod.rs index 4a168c81..69e93a5a 100644 --- a/ant-core/tests/support/mod.rs +++ b/ant-core/tests/support/mod.rs @@ -19,10 +19,12 @@ use ant_core::data::ClientConfig; // Node-internal types (test harness needs to *be* a node) — direct // ant-node import is correct here. ant-node is a dev-dep so this is // only linked into test binaries. +use ant_node::payment::quote::CommitmentSource; use ant_node::payment::{ EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, QuoteGenerator, QuotingMetricsTracker, }; +use ant_node::replication::commitment_state::{BuiltCommitment, ResponderCommitmentState}; use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; // Wire / transport / EVM types: route through ant-protocol so the test // harness exercises the same surface the client does. @@ -111,7 +113,21 @@ impl MiniTestnet { /// Start a testnet with the given number of nodes. /// /// Use `DEFAULT_NODE_COUNT` for standard tests, 35+ for merkle tests (need 16 peers per pool). + /// Nodes emit baseline `(0, None)` quotes (no commitment source wired). pub async fn start(node_count: usize) -> Self { + Self::start_inner(node_count, None).await + } + + /// ADR-0003: start a testnet where every node carries a live storage + /// commitment over `key_count` synthetic keys, so they emit COMMITMENT-BOUND + /// quotes (price = `calculate_price(key_count)`, pinned, commitment shipped + /// in the quote response). Exercises the full node→client→storer binding + /// handshake against real QUIC + Anvil. + pub async fn start_with_commitments(node_count: usize, key_count: u32) -> Self { + Self::start_inner(node_count, Some(key_count)).await + } + + async fn start_inner(node_count: usize, commitment_key_count: Option) -> Self { // Start Anvil EVM testnet FIRST let testnet = Testnet::new().await.expect("start Anvil testnet"); let evm_network = testnet.to_network(); @@ -136,8 +152,15 @@ impl MiniTestnet { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let (node, protocol, handler) = - Self::spawn_node(addr, &bootstrap_addrs, temp_dir.path(), &evm_network, i).await; + let (node, protocol, handler) = Self::spawn_node( + addr, + &bootstrap_addrs, + temp_dir.path(), + &evm_network, + i, + commitment_key_count, + ) + .await; bootstrap_addrs.push(addr); nodes.push(TestNode { @@ -155,8 +178,15 @@ impl MiniTestnet { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let (node, protocol, handler) = - Self::spawn_node(addr, &bootstrap_addrs, temp_dir.path(), &evm_network, i).await; + let (node, protocol, handler) = Self::spawn_node( + addr, + &bootstrap_addrs, + temp_dir.path(), + &evm_network, + i, + commitment_key_count, + ) + .await; nodes.push(TestNode { p2p_node: Some(Arc::clone(&node)), @@ -261,6 +291,7 @@ impl MiniTestnet { data_dir: &std::path::Path, evm_network: &EvmNetwork, node_index: usize, + commitment_key_count: Option, ) -> (Arc, Arc, tokio::task::JoinHandle<()>) { // Generate ML-DSA-65 identity for this node let identity = Arc::new(NodeIdentity::generate().expect("generate node identity")); @@ -342,6 +373,16 @@ impl MiniTestnet { // and payment closeness checks use the node's live DHT view. protocol.attach_p2p_node(Arc::clone(&node)); + // ADR-0003: optionally give this node a live storage commitment so it + // emits COMMITMENT-BOUND quotes (price = calculate_price(key_count), + // pinned, with the signed commitment shipped in the quote response). + // Without this the node has no commitment source and emits baseline + // `(0, None)` quotes — the path the rest of the suite exercises. + if let Some(key_count) = commitment_key_count { + let source = build_commitment_source(key_count, &identity); + protocol.attach_commitment_source(source); + } + // Start message handler loop let handler_node = Arc::clone(&node); let handler_protocol = Arc::clone(&protocol); @@ -441,3 +482,42 @@ impl MiniTestnet { } } } + +/// ADR-0003: build a live `ResponderCommitmentState` holding one current +/// commitment over `key_count` synthetic keys, signed by `identity`'s ML-DSA-65 +/// key and bound to its peer id (`BLAKE3(pub_key)`). Returned as the +/// `CommitmentSource` the quote generator prices against — so the node emits a +/// commitment-bound quote (`calculate_price(key_count)`, pinned) and ships the +/// signed commitment in the quote response. The commitment is real: it passes +/// the storer's signature, peer-binding, and hash==pin checks. +fn build_commitment_source(key_count: u32, identity: &NodeIdentity) -> Arc { + // Use the SAME identity→commitment-key conversion the replication engine + // uses in production (`MlDsaSecretKey::from_bytes(MlDsa65, identity bytes)`), + // so the commitment is signed by the node's real key and binds to its real + // peer id — exactly what the storer's cross-check verifies. + use ant_protocol::pqc::api::{MlDsaSecretKey, MlDsaVariant}; + + let pub_key = identity.public_key().as_bytes().to_vec(); + let peer_id = *blake3::hash(&pub_key).as_bytes(); + let sk_bytes = identity.secret_key_bytes().to_vec(); + let sk = MlDsaSecretKey::from_bytes(MlDsaVariant::MlDsa65, &sk_bytes) + .expect("deserialize ML-DSA-65 secret key"); + + // `key_count` distinct (key, bytes_hash) leaves — the commitment attests + // exactly this many keys, which becomes the quote's committed_key_count. + let entries: Vec<([u8; 32], [u8; 32])> = (0..key_count) + .map(|i| { + let mut k = [0u8; 32]; + k[..4].copy_from_slice(&i.to_le_bytes()); + let mut b = [1u8; 32]; + b[..4].copy_from_slice(&i.to_le_bytes()); + (k, b) + }) + .collect(); + + let built = + BuiltCommitment::build(entries, &peer_id, &sk, &pub_key).expect("build storage commitment"); + let state = ResponderCommitmentState::new(); + state.rotate(built); + Arc::new(state) +} From e1439ee577e3b481e1f4352c27c33e3cfb164064 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 30 Jun 2026 17:41:31 +0200 Subject: [PATCH 12/22] =?UTF-8?q?chore:=20TEMP=20ADR-0004=20git=20deps=20f?= =?UTF-8?q?or=20CI=20=E2=80=94=20STRIP=20BEFORE=20MERGE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [patch.crates-io] points evmlib/ant-protocol at their PR branches; ant-core points the ant-node dep (runtime + dev) at the matching ant-node ADR-0004 branch (published versions version-collide with the patch). At release: revert ant-node to a published version, drop the patch block, bump ant-protocol pin. --- Cargo.lock | 23 ++++++++----------- Cargo.toml | 18 +++++++++++++++ ant-core/Cargo.toml | 6 ++--- .../tests/{e2e_adr0003.rs => e2e_adr0004.rs} | 0 4 files changed, 31 insertions(+), 16 deletions(-) rename ant-core/tests/{e2e_adr0003.rs => e2e_adr0004.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index 86093ef3..bfa5ef91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -815,7 +815,7 @@ dependencies = [ [[package]] name = "ant-cli" -version = "0.2.10" +version = "0.2.8" dependencies = [ "ant-core", "anyhow", @@ -835,7 +835,7 @@ dependencies = [ [[package]] name = "ant-core" -version = "0.3.0" +version = "0.2.8" dependencies = [ "alloy", "ant-node", @@ -892,9 +892,8 @@ dependencies = [ [[package]] name = "ant-node" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbbc1f23610ffb700cd85bbf7519ebe29c767ccffc85480c29277da627de41de" +version = "0.14.1" +source = "git+https://github.com/grumbach/ant-node?branch=adr-0003-commitment-bound-pricing#195981f53a92f1a6522f8bf96c903c2259c50a09" dependencies = [ "ant-protocol", "blake3", @@ -942,9 +941,8 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a35986649c5963ec808efb44621d8b94716cc2ef5a041ea9a1875487578097c" +version = "2.2.1" +source = "git+https://github.com/WithAutonomi/ant-protocol?branch=adr-0003-commitment-bound-pricing#140adea15c4f7e3594d13a017a0770677171322b" dependencies = [ "blake3", "bytes", @@ -2530,8 +2528,7 @@ dependencies = [ [[package]] name = "evmlib" version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd96cf24017cd31cd422c80de3078a821b7884a5b1feb00087e98209326c676" +source = "git+https://github.com/WithAutonomi/evmlib?branch=adr-0003-signed-quote-fields#4adfdb5615271f716e4233681fb4cf640a0a34d0" dependencies = [ "alloy", "ant-merkle", @@ -3261,7 +3258,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.57.0", + "windows-core 0.58.0", ] [[package]] @@ -5250,9 +5247,9 @@ dependencies = [ [[package]] name = "saorsa-core" -version = "0.26.2" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d9c05644c46b8de670aa5c8eea037b90731f1f204e8aa5d8a2f03496d740401" +checksum = "c3f44ef51271c8bfc5b5305f73831a5090fde070553e2a96636feae109b4fdc4" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index f3978b22..745a1102 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,21 @@ [workspace] members = ["ant-core", "ant-cli"] resolver = "2" + +# ─── TEMPORARY: STRIP BEFORE MERGE — ADR-0004 coordinated cutover ─── +# ant-protocol/evmlib's ADR-0004 commitment-bound-quote types (the signed +# `committed_key_count`/`commitment_pin` fields, the commitment sidecar, and +# `BadQuoteCommitment`) are not yet published to crates.io, so this workspace +# cannot build against the published `ant-protocol`/`evmlib`. These git patches +# point those deps at their PR branches so CI can build the coordinated change. +# +# AT RELEASE (once the ADR-0004 versions are published, in order +# evmlib → ant-protocol): +# 1. delete this entire [patch.crates-io] block, AND +# 2. bump the `ant-protocol` pin in ant-core/Cargo.toml to the published +# version (evmlib is re-exported through ant-protocol, no direct pin): +# ant-protocol = "2.3.0" # <- published ADR-0004 ant-protocol +# # (which in turn depends on the published evmlib = "0.9.0") +[patch.crates-io] +evmlib = { git = "https://github.com/WithAutonomi/evmlib", branch = "adr-0003-signed-quote-fields" } +ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", branch = "adr-0003-commitment-bound-pricing" } diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 36973729..59ea1ce9 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -37,7 +37,7 @@ tower-http = { version = "0.6.8", features = ["cors"] } # under `ant_protocol::{evm, transport, pqc}`. This is the ONE pin for # those three deps — do not add direct evmlib/saorsa-core/saorsa-pqc # deps here or the version can skew between ant-client and ant-node. -ant-protocol = "2.2.2" +ant-protocol = "2.3.0" xor_name = "5" self_encryption = "0.36" futures = "0.3" @@ -65,7 +65,7 @@ sysinfo = { version = "0.32", default-features = false, features = ["system"] } # `ant-protocol` pin above points at a git branch, this ant-node must point at # the matching ant-node branch carrying the same saorsa-core / ant-protocol # lineage rather than a released version. -ant-node = { version = "0.14.2", optional = true } +ant-node = { git = "https://github.com/grumbach/ant-node", branch = "adr-0003-commitment-bound-pricing", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter"] } [target.'cfg(unix)'.dependencies] @@ -96,7 +96,7 @@ test-utils = [] # always compile even without the `devnet` feature. Pinned to the same # version as the runtime dep so there is a single ant-node / # saorsa-core version across the whole graph. -ant-node = { version = "0.14.2", features = ["test-utils"] } +ant-node = { git = "https://github.com/grumbach/ant-node", branch = "adr-0003-commitment-bound-pricing", features = ["test-utils"] } serial_test = "3" anyhow = "1" alloy = { version = "1.6", features = ["node-bindings"] } diff --git a/ant-core/tests/e2e_adr0003.rs b/ant-core/tests/e2e_adr0004.rs similarity index 100% rename from ant-core/tests/e2e_adr0003.rs rename to ant-core/tests/e2e_adr0004.rs From eec61c81e0b05f30f956bc2cf0aa369b144ea26b Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 30 Jun 2026 17:41:31 +0200 Subject: [PATCH 13/22] feat(client): thread ADR-0004 commitment sidecar through witnessed quote flow; renumber ADR-0003 -> ADR-0004 StoreQuote carries the commitment sidecar end-to-end (resolve-before-pay); get_store_quotes family + median helpers + downstream consumers updated. --- ant-core/src/data/client/batch.rs | 6 +-- ant-core/src/data/client/file.rs | 9 +++- ant-core/src/data/client/merkle.rs | 14 +++--- ant-core/src/data/client/payment.rs | 2 +- ant-core/src/data/client/quote.rs | 77 ++++++++++++++++------------- ant-core/src/data/error.rs | 2 +- ant-core/tests/e2e_adr0004.rs | 10 ++-- ant-core/tests/support/mod.rs | 6 +-- 8 files changed, 72 insertions(+), 54 deletions(-) diff --git a/ant-core/src/data/client/batch.rs b/ant-core/src/data/client/batch.rs index f68e1808..a30f67b3 100644 --- a/ant-core/src/data/client/batch.rs +++ b/ant-core/src/data/client/batch.rs @@ -53,7 +53,7 @@ pub struct PreparedChunk { pub payment: SingleNodePayment, /// Peer quotes for building `ProofOfPayment`. pub peer_quotes: Vec<(EncodedPeerId, PaymentQuote)>, - /// ADR-0003: the signed commitments the bound quotes shipped, forwarded as + /// ADR-0004: the signed commitments the bound quotes shipped, forwarded as /// sidecars in the PUT bundle so storers cross-check synchronously. Empty /// when every quote was baseline (no commitment to pin). pub commitment_sidecars: Vec>, @@ -214,7 +214,7 @@ fn build_paid_chunks( peer_quotes: chunk.peer_quotes, }, tx_hashes, - // ADR-0003: forward the bound quotes' commitments so storers + // ADR-0004: forward the bound quotes' commitments so storers // cross-check synchronously; stripped before persistence node-side. commitment_sidecars: chunk.commitment_sidecars, }; @@ -279,7 +279,7 @@ impl Client { // Use node-reported prices directly — no contract price fetch needed. let mut peer_quotes = Vec::with_capacity(quotes_with_peers.len()); let mut quotes_for_payment = Vec::with_capacity(quotes_with_peers.len()); - // ADR-0003: forward each bound quote's commitment sidecar (baseline + // ADR-0004: forward each bound quote's commitment sidecar (baseline // quotes ship none); `get_store_quotes` already verified the binding. let mut commitment_sidecars = Vec::new(); diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index 84737ee1..960399fd 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -139,7 +139,7 @@ pub enum FileChunkPeerStatus { /// One entry in the per-chunk quote list returned by /// [`Client::get_store_quotes`]: the responding peer, its addresses, the -/// signed quote it returned, the payment amount it is demanding, and (ADR-0003) +/// signed quote it returned, the payment amount it is demanding, and (ADR-0004) /// the opaque signed-commitment blob the node shipped with the quote. type QuoteEntry = ( PeerId, @@ -977,6 +977,13 @@ pub struct FileUploadResult { } /// Payment information for external signing — either wave-batch or merkle. +// ADR-0004 added the signed commitment fields (`committed_key_count`, +// `commitment_pin`) to the merkle candidate quotes carried inside +// `PreparedMerkleBatch`, which grew the `Merkle` variant past the +// `large_enum_variant` threshold. This enum is constructed one-off per payment +// (never held in bulk collections), so the size delta is harmless; allow it +// rather than box a field on the security-sensitive merkle-finalize path. +#[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum ExternalPaymentInfo { /// Wave-batch: individual (quote_hash, rewards_address, amount) tuples. diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index 08700950..091a3f6d 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -37,7 +37,7 @@ use xor_name::XorName; /// Default threshold: use merkle payments when chunk count >= this value. pub const DEFAULT_MERKLE_THRESHOLD: usize = 64; -/// ADR-0003 resolve-before-pay gate for a merkle candidate — the merkle-path +/// ADR-0004 resolve-before-pay gate for a merkle candidate — the merkle-path /// equivalent of the single-node `quote_commitment_binding_is_valid`. Runs the /// FULL binding check (shape, cap, exact price, and for bound candidates the /// commitment parse, peer-binding, signature, `hash == pin`, and @@ -163,7 +163,7 @@ pub struct PreparedMerkleBatch { tree: MerkleTree, /// Internal: chunk addresses in order. addresses: Vec<[u8; 32]>, - /// ADR-0003: validated commitment sidecars keyed by `(peer, pin)`, collected + /// ADR-0004: validated commitment sidecars keyed by `(peer, pin)`, collected /// during candidate validation. At proof time the winner pool's candidates /// forward theirs in `MerklePaymentProof.commitment_sidecars`. commitment_sidecars: HashMap<(PeerId, [u8; 32]), Vec>, @@ -528,7 +528,7 @@ impl Client { ); // 3. Collect candidate pools from the network (all pools in parallel). - // Each candidate's ADR-0003 binding is verified during collection and + // Each candidate's ADR-0004 binding is verified during collection and // its commitment sidecar captured (keyed by peer, pin). let (candidate_pools, commitment_sidecars) = self .build_candidate_pools( @@ -695,7 +695,7 @@ impl Client { } let mut pools = Vec::with_capacity(midpoint_proofs.len()); - // ADR-0003: merged map of every validated candidate's commitment sidecar, + // ADR-0004: merged map of every validated candidate's commitment sidecar, // keyed by (peer, pin). At proof time the winner pool's candidates look // up their sidecars here to forward in the merkle PUT bundle. let mut sidecars: HashMap<(PeerId, [u8; 32]), Vec> = HashMap::new(); @@ -888,7 +888,7 @@ impl Client { failures.push(format!("{peer_id}: candidate pub_key/peer mismatch")); continue; } - // ADR-0003: the FULL resolve-before-pay binding check, same as + // ADR-0004: the FULL resolve-before-pay binding check, same as // the single-node path — a candidate priced off its committed // count, or shipping an unresolvable/forged commitment, is // dropped before it can enter a pool the client pays. Checked @@ -896,7 +896,7 @@ impl Client { if let Err(detail) = merkle_candidate_binding_is_valid(&candidate_peer, &candidate, &commitment) { - warn!("Dropping merkle candidate {peer_id} — ADR-0003 binding invalid: {detail}"); + warn!("Dropping merkle candidate {peer_id} — ADR-0004 binding invalid: {detail}"); failures.push(format!("{peer_id}: bad commitment binding ({detail})")); continue; } @@ -1446,7 +1446,7 @@ pub fn finalize_merkle_batch( )) })?; - // ADR-0003: collect the winner pool's candidate commitment sidecars (those + // ADR-0004: collect the winner pool's candidate commitment sidecars (those // that were bound + validated during collection), to forward in each proof // so the storer can cross-check synchronously. Built once for the pool. let winner_sidecars: Vec> = winner_pool diff --git a/ant-core/src/data/client/payment.rs b/ant-core/src/data/client/payment.rs index 3d2fb3e7..859ded4a 100644 --- a/ant-core/src/data/client/payment.rs +++ b/ant-core/src/data/client/payment.rs @@ -68,7 +68,7 @@ impl Client { // Use node-reported prices directly — no contract price fetch needed. let mut peer_quotes = Vec::with_capacity(quotes_with_peers.len()); let mut quotes_for_payment = Vec::with_capacity(quotes_with_peers.len()); - // ADR-0003: forward the signed commitment each bound quote shipped, so + // ADR-0004: forward the signed commitment each bound quote shipped, so // the storers can cross-check the quote's count against the original // commitment synchronously ("the commitment arrived with the quote"). // A baseline quote ships none. `get_store_quotes` already verified each diff --git a/ant-core/src/data/client/quote.rs b/ant-core/src/data/client/quote.rs index 2737f191..4d9d29d4 100644 --- a/ant-core/src/data/client/quote.rs +++ b/ant-core/src/data/client/quote.rs @@ -61,7 +61,7 @@ const QUOTE_COLLECTION_TIMEOUT_SECS: u64 = 120; const ML_DSA_PUB_KEY_LEN: usize = 1952; /// One collected quote: the responding peer, its addresses, the signed quote, -/// the price it demands, and (ADR-0003) the opaque signed-commitment blob the +/// the price it demands, and (ADR-0004) the opaque signed-commitment blob the /// node shipped alongside the quote (`None` for a baseline quote), to be /// forwarded as a sidecar in the PUT bundle. type QuotedPeer = ( @@ -72,9 +72,6 @@ type QuotedPeer = ( Option>, ); -/// Per-peer classification outcome: `(quote, price, commitment)` on success. -type QuoteResult = std::result::Result<(PaymentQuote, Amount, Option>), Error>; - /// Check that a quote's `pub_key` is well-formed and BLAKE3-hashes to the /// claimed `peer_id`. /// @@ -85,7 +82,7 @@ type QuoteResult = std::result::Result<(PaymentQuote, Amount, Option>), /// failing either check causes the storer to reject the entire close-group /// proof and burn the chunk's payment. /// -/// This is the cheap structural pre-check. ADR-0003 additionally has the client +/// This is the cheap structural pre-check. ADR-0004 additionally has the client /// run `verify_quote_content` + `verify_quote_signature` (the full ML-DSA check) /// in [`classify_quote_response`] before paying, so a quote the storer would /// reject never gets paid. @@ -96,7 +93,7 @@ fn quote_binding_is_valid(peer_id: &PeerId, quote: &PaymentQuote) -> bool { compute_address("e.pub_key) == *peer_id.as_bytes() } -/// ADR-0003 client-side resolve-before-pay gate — "the client pays nothing it +/// ADR-0004 client-side resolve-before-pay gate — "the client pays nothing it /// cannot resolve", the ceiling's load-bearing wall (ADR §"The client pays /// nothing it cannot resolve"). /// @@ -206,7 +203,7 @@ fn quote_commitment_binding_is_valid( /// `ant-node/src/payment/verifier.rs` — the cheap BLAKE3 binding — /// so we drop misbehaving peers' quotes before payment. /// -/// ADR-0003: the client now ALSO runs the storer's `verify_quote_content` and +/// ADR-0004: the client now ALSO runs the storer's `verify_quote_content` and /// `verify_quote_signature` (ML-DSA-65) before paying, so "the client pays /// nothing it cannot resolve" covers the quote's own validity too, not just the /// commitment binding. This matches what the merkle path already does @@ -224,7 +221,7 @@ fn quote_commitment_binding_is_valid( /// - `Err(Error::BadQuoteBinding { .. })` — bad binding (mirrors the /// storer-side rejection). Outer collector counts these via the typed /// variant (no string matching). -/// - `Err(Error::BadQuoteCommitment { .. })` — ADR-0003 forced-price binding +/// - `Err(Error::BadQuoteCommitment { .. })` — ADR-0004 forced-price binding /// failed (price off the curve, incoherent shape, or a bound quote that did /// not ship its commitment); dropped before payment like a bad binding. /// - `Err(Error::Serialization(...))` — the quote bytes did not deserialize. @@ -264,7 +261,7 @@ fn classify_quote_response( }); } - // ADR-0003 "the client runs the full binding check": verify the quote's OWN + // ADR-0004 "the client runs the full binding check": verify the quote's OWN // ML-DSA-65 signature and that it is for THIS content, before paying — // exactly what the storer checks and what the merkle path already does // client-side. A quote with a valid pub_key binding but a bad signature or @@ -282,13 +279,13 @@ fn classify_quote_response( }); } - // ADR-0003 forced-price gate: drop a quote whose price is not exactly the + // ADR-0004 forced-price gate: drop a quote whose price is not exactly the // public formula of its committed count, whose (count, pin) shape is // incoherent, or which is bound but did not ship its commitment. The storer // re-runs the arithmetic and would reject the bundle; we drop it here so we // never pay a quote we cannot resolve. if let Err(detail) = quote_commitment_binding_is_valid(peer_id, &payment_quote, &commitment) { - warn!("Dropping response from {peer_id} — ADR-0003 binding invalid: {detail}"); + warn!("Dropping response from {peer_id} — ADR-0004 binding invalid: {detail}"); return Err(Error::BadQuoteCommitment { peer_id: peer_id.to_string(), detail, @@ -368,7 +365,14 @@ async fn request_store_quote_from_peer( ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Success { quote, already_stored, - }) => Some(classify_quote_response(&peer_id, "e, already_stored)), + commitment, + }) => Some(classify_quote_response( + &peer_id, + &address, + "e, + already_stored, + commitment, + )), ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error(e)) => Some(Err( Error::Protocol(format!("Quote error from {peer_id}: {e}")), )), @@ -386,7 +390,7 @@ async fn request_store_quote_from_peer( fn record_store_quote_result( peer_id: PeerId, addrs: Vec, - quote_result: Result<(PaymentQuote, Amount)>, + quote_result: Result<(PaymentQuote, Amount, Option>)>, address: &[u8; 32], quotes: &mut Vec, already_stored_peers: &mut Vec<(PeerId, [u8; 32])>, @@ -394,8 +398,8 @@ fn record_store_quote_result( bad_quote_count: &mut usize, ) { match quote_result { - Ok((quote, price)) => { - quotes.push((peer_id, addrs, quote, price)); + Ok((quote, price, commitment)) => { + quotes.push((peer_id, addrs, quote, price, commitment)); } Err(Error::AlreadyStored) => { info!("Peer {peer_id} reports chunk already stored"); @@ -485,8 +489,12 @@ fn peer_list(peers: &[PeerId]) -> Vec { peers.iter().map(ToString::to_string).collect() } -pub(crate) type StoreQuote = (PeerId, Vec, PaymentQuote, Amount); -type StoreQuoteRequestResult = (PeerId, Vec, Result<(PaymentQuote, Amount)>); +/// One collected store quote, carrying (ADR-0004) the opaque signed-commitment +/// sidecar the node shipped with its quote (`None` for a baseline quote), to be +/// forwarded in the PUT bundle and cross-checked by storers. +pub(crate) type StoreQuote = (PeerId, Vec, PaymentQuote, Amount, Option>); +type StoreQuoteRequestResult = + (PeerId, Vec, Result<(PaymentQuote, Amount, Option>)>); type VotersByPeer = HashMap>; type WitnessedVoteData = (HashMap, VotersByPeer, Vec<(PeerId, usize)>); @@ -720,9 +728,7 @@ fn witnessed_quote_selection_or_error( }) } -pub(crate) fn median_paid_quote_issuer( - quotes: &[(PeerId, Vec, PaymentQuote, Amount)], -) -> Option<(PeerId, Amount)> { +pub(crate) fn median_paid_quote_issuer(quotes: &[StoreQuote]) -> Option<(PeerId, Amount)> { if quotes.len() <= MEDIAN_QUOTE_INDEX { return None; } @@ -730,7 +736,7 @@ pub(crate) fn median_paid_quote_issuer( let mut by_price: Vec<(usize, PeerId, Amount)> = quotes .iter() .enumerate() - .map(|(index, (peer_id, _, _, price))| (index, *peer_id, *price)) + .map(|(index, (peer_id, _, _, price, _))| (index, *peer_id, *price)) .collect(); by_price.sort_by_key(|(index, _, price)| (*price, *index)); by_price @@ -758,7 +764,7 @@ fn median_paid_quote_issuer_for_indices( .iter() .enumerate() .map(|(selected_index, quote_index)| { - let (peer_id, _, _, price) = "es[*quote_index]; + let (peer_id, _, _, price, _) = "es[*quote_index]; (selected_index, *peer_id, *price) }) .collect(); @@ -900,7 +906,7 @@ impl Client { address: &[u8; 32], data_size: u64, data_type: u32, - ) -> Result, PaymentQuote, Amount)>> { + ) -> Result> { Ok(self .get_store_quote_plan(address, data_size, data_type) .await? @@ -974,7 +980,7 @@ impl Client { address: &[u8; 32], data_size: u64, data_type: u32, - ) -> Result, PaymentQuote, Amount)>> { + ) -> Result> { let peer_query_count = fault_tolerant_quote_query_count(); let remote_peers = self .network() @@ -1085,7 +1091,7 @@ impl Client { data_type: u32, remote_peers: Vec<(PeerId, Vec)>, quote_selection_policy: QuoteSelectionPolicy, - ) -> Result, PaymentQuote, Amount)>> { + ) -> Result> { let peer_query_count = remote_peers.len(); let node = self.network().node(); @@ -1388,7 +1394,7 @@ mod tests { /// Build a quote tuple whose `pub_key` correctly hashes to its peer_id. /// Signature is left empty: this filter does not verify signatures. /// - /// The quote is a valid ADR-0003 **baseline**: `(0, None)` priced at + /// The quote is a valid ADR-0004 **baseline**: `(0, None)` priced at /// `calculate_price(0)`, so it passes the forced-price gate in /// `classify_quote_response`. The 5th tuple element is the (absent) /// commitment sidecar. @@ -1454,7 +1460,10 @@ mod tests { PeerId::from_bytes([seed; 32]) } - fn synthetic_quote(seed: u8, price: u64) -> (PeerId, Vec, PaymentQuote, Amount) { + fn synthetic_quote( + seed: u8, + price: u64, + ) -> (PeerId, Vec, PaymentQuote, Amount, Option>) { let amount = Amount::from(price); let quote = PaymentQuote { content: XorName([0u8; 32]), @@ -1463,18 +1472,20 @@ mod tests { rewards_address: RewardsAddress::new([0u8; 20]), pub_key: Vec::new(), signature: Vec::new(), + committed_key_count: 0, + commitment_pin: None, }; - (synthetic_peer(seed), Vec::new(), quote, amount) + (synthetic_peer(seed), Vec::new(), quote, amount, None) } fn synthetic_voters(seeds: &[u8]) -> HashSet { seeds.iter().copied().map(synthetic_peer).collect() } - fn quote_peer_seeds(quotes: &[(PeerId, Vec, PaymentQuote, Amount)]) -> Vec { + fn quote_peer_seeds(quotes: &[StoreQuote]) -> Vec { quotes .iter() - .map(|(peer_id, _, _, _)| peer_id.as_bytes()[0]) + .map(|(peer_id, _, _, _, _)| peer_id.as_bytes()[0]) .collect() } @@ -1950,7 +1961,7 @@ mod tests { median_paid_quote_issuer(&selected).expect("selected quotes have a median"); let selected_peers = selected .iter() - .map(|(peer_id, _, _, _)| *peer_id) + .map(|(peer_id, _, _, _, _)| *peer_id) .collect::>(); assert_eq!(median_peer_id, synthetic_peer(MEDIAN_ISSUER_SEED)); assert_eq!( @@ -2371,7 +2382,7 @@ mod tests { } // ============================================================ - // ADR-0003: quote_commitment_binding_is_valid (forced-price gate) + // ADR-0004: quote_commitment_binding_is_valid (forced-price gate) // // Mirrors the storer-side `binding_violation` in // `ant-node/src/payment/verifier.rs`. The client runs this before @@ -2380,7 +2391,7 @@ mod tests { // and for bound quotes: parse + peer-binding + signature + hash==pin + // count==key_count) using the shared ant-protocol commitment type, so an // unresolvable/forged commitment is never paid. A live resolve against a - // REAL signed commitment is proven in the e2e suite (e2e_adr0003.rs). + // REAL signed commitment is proven in the e2e suite (e2e_adr0004.rs). // ============================================================ /// A throwaway peer id for tests that fail BEFORE commitment resolution diff --git a/ant-core/src/data/error.rs b/ant-core/src/data/error.rs index 8f7ed960..e0f49b7f 100644 --- a/ant-core/src/data/error.rs +++ b/ant-core/src/data/error.rs @@ -119,7 +119,7 @@ pub enum Error { detail: String, }, - /// ADR-0003: a quote's commitment binding does not hold — its price is not + /// ADR-0004: a quote's commitment binding does not hold — its price is not /// `calculate_price(committed_key_count)`, its `(count, pin)` shape is /// incoherent, or a shipped commitment does not match the pinned count/hash. /// The storer's arithmetic gate would reject such a quote, so the client diff --git a/ant-core/tests/e2e_adr0004.rs b/ant-core/tests/e2e_adr0004.rs index 448047c4..43121e1b 100644 --- a/ant-core/tests/e2e_adr0004.rs +++ b/ant-core/tests/e2e_adr0004.rs @@ -1,11 +1,11 @@ -//! ADR-0003 end-to-end: commitment-bound quote pricing. +//! ADR-0004 end-to-end: commitment-bound quote pricing. //! //! Proves, against a real in-process QUIC testnet + Anvil EVM, that: //! //! 1. A node carrying a live storage commitment emits a COMMITMENT-BOUND quote: //! `committed_key_count == N`, `commitment_pin == Some`, the price is exactly //! `calculate_price(N)`, and the signed commitment is SHIPPED in the quote -//! response (ADR-0003 "the commitment arrived with the quote"). +//! response (ADR-0004 "the commitment arrived with the quote"). //! 2. The client's forced-price gate ACCEPTS those bound quotes (they are //! self-consistent) and a full pay → store → retrieve round-trip succeeds — //! so the storer accepts the bound quotes and the forwarded sidecars too. @@ -39,7 +39,7 @@ const COMMITTED_KEYS: u32 = 9_000; #[tokio::test(flavor = "multi_thread")] #[serial] -async fn adr0003_bound_quotes_are_shipped_priced_and_resolve() { +async fn adr0004_bound_quotes_are_shipped_priced_and_resolve() { let testnet = MiniTestnet::start_with_commitments(DEFAULT_NODE_COUNT, COMMITTED_KEYS).await; let node = testnet.node(3).expect("node 3 exists"); let client = Client::from_node(Arc::clone(&node), test_client_config()) @@ -48,7 +48,7 @@ async fn adr0003_bound_quotes_are_shipped_priced_and_resolve() { let content = Bytes::from("adr-0003 bound-quote payload"); let address = compute_address(&content); - // Collect quotes directly so we can inspect the ADR-0003 binding the client + // Collect quotes directly so we can inspect the ADR-0004 binding the client // verified before it would pay. let quotes = client .get_store_quotes(&address, content.len() as u64, 0) @@ -141,7 +141,7 @@ async fn adr0003_bound_quotes_are_shipped_priced_and_resolve() { /// This guards the baseline branch of the same forced-price gate. #[tokio::test(flavor = "multi_thread")] #[serial] -async fn adr0003_baseline_quotes_still_work() { +async fn adr0004_baseline_quotes_still_work() { let testnet = MiniTestnet::start(DEFAULT_NODE_COUNT).await; let node = testnet.node(3).expect("node 3 exists"); let client = Client::from_node(Arc::clone(&node), test_client_config()) diff --git a/ant-core/tests/support/mod.rs b/ant-core/tests/support/mod.rs index 69e93a5a..9486fecc 100644 --- a/ant-core/tests/support/mod.rs +++ b/ant-core/tests/support/mod.rs @@ -118,7 +118,7 @@ impl MiniTestnet { Self::start_inner(node_count, None).await } - /// ADR-0003: start a testnet where every node carries a live storage + /// ADR-0004: start a testnet where every node carries a live storage /// commitment over `key_count` synthetic keys, so they emit COMMITMENT-BOUND /// quotes (price = `calculate_price(key_count)`, pinned, commitment shipped /// in the quote response). Exercises the full node→client→storer binding @@ -373,7 +373,7 @@ impl MiniTestnet { // and payment closeness checks use the node's live DHT view. protocol.attach_p2p_node(Arc::clone(&node)); - // ADR-0003: optionally give this node a live storage commitment so it + // ADR-0004: optionally give this node a live storage commitment so it // emits COMMITMENT-BOUND quotes (price = calculate_price(key_count), // pinned, with the signed commitment shipped in the quote response). // Without this the node has no commitment source and emits baseline @@ -483,7 +483,7 @@ impl MiniTestnet { } } -/// ADR-0003: build a live `ResponderCommitmentState` holding one current +/// ADR-0004: build a live `ResponderCommitmentState` holding one current /// commitment over `key_count` synthetic keys, signed by `identity`'s ML-DSA-65 /// key and bound to its peer id (`BLAKE3(pub_key)`). Returned as the /// `CommitmentSource` the quote generator prices against — so the node emits a From 4302c64ceab63bc21ad9e098ccc3638fa0e233b7 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 30 Jun 2026 17:48:16 +0200 Subject: [PATCH 14/22] fix(ci): rustfmt + bump quinn-proto 0.11.14 -> 0.11.15 (RUSTSEC-2026-0185) --- Cargo.lock | 4 ++-- ant-core/src/data/client/quote.rs | 23 +++++++++++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfa5ef91..c79afbcf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4576,9 +4576,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", diff --git a/ant-core/src/data/client/quote.rs b/ant-core/src/data/client/quote.rs index 4d9d29d4..aa870377 100644 --- a/ant-core/src/data/client/quote.rs +++ b/ant-core/src/data/client/quote.rs @@ -492,9 +492,18 @@ fn peer_list(peers: &[PeerId]) -> Vec { /// One collected store quote, carrying (ADR-0004) the opaque signed-commitment /// sidecar the node shipped with its quote (`None` for a baseline quote), to be /// forwarded in the PUT bundle and cross-checked by storers. -pub(crate) type StoreQuote = (PeerId, Vec, PaymentQuote, Amount, Option>); -type StoreQuoteRequestResult = - (PeerId, Vec, Result<(PaymentQuote, Amount, Option>)>); +pub(crate) type StoreQuote = ( + PeerId, + Vec, + PaymentQuote, + Amount, + Option>, +); +type StoreQuoteRequestResult = ( + PeerId, + Vec, + Result<(PaymentQuote, Amount, Option>)>, +); type VotersByPeer = HashMap>; type WitnessedVoteData = (HashMap, VotersByPeer, Vec<(PeerId, usize)>); @@ -1463,7 +1472,13 @@ mod tests { fn synthetic_quote( seed: u8, price: u64, - ) -> (PeerId, Vec, PaymentQuote, Amount, Option>) { + ) -> ( + PeerId, + Vec, + PaymentQuote, + Amount, + Option>, + ) { let amount = Amount::from(price); let quote = PaymentQuote { content: XorName([0u8; 32]), From 9b93ad9b206d2129c4784d22a7e17bcdfeb72628 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 1 Jul 2026 12:56:00 +0200 Subject: [PATCH 15/22] ci: run e2e_adr0004 in the E2E job The renamed ADR-0004 e2e target was auto-discovered but never listed in the CI E2E command, so ant-client could go green without exercising the main ADR-0004 end-to-end coverage. Add --test e2e_adr0004. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d22d5761..2382b67c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: anvil --version forge --version - name: Run E2E tests (serial) - run: cargo test -p ant-core --features test-utils --test e2e_chunk --test e2e_data --test e2e_file --test e2e_payment --test e2e_security --test e2e_cost_estimate -- --test-threads=1 + run: cargo test -p ant-core --features test-utils --test e2e_chunk --test e2e_data --test e2e_file --test e2e_payment --test e2e_security --test e2e_cost_estimate --test e2e_adr0004 -- --test-threads=1 test-merkle: name: Merkle E2E (${{ matrix.os }}) From d676532401faad26862d47aa4a24b901de6f0ce5 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 1 Jul 2026 15:37:59 +0200 Subject: [PATCH 16/22] chore: bump ADR-0004 git deps to clean-rebased revs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point the ant-node / ant-protocol / evmlib git-branch deps at the freshly clean-rebased ADR-0004 revisions (ant-node ff7a547 — the audit-code merge against current main resolved correctly; ant-protocol 3a4eb638 = 2.3.0; evmlib a7ac7e26 = 0.9.0). Recovers the workspace build now that ant-node's branch compiles again. --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c79afbcf..faf38033 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -815,7 +815,7 @@ dependencies = [ [[package]] name = "ant-cli" -version = "0.2.8" +version = "0.2.9" dependencies = [ "ant-core", "anyhow", @@ -835,7 +835,7 @@ dependencies = [ [[package]] name = "ant-core" -version = "0.2.8" +version = "0.2.9" dependencies = [ "alloy", "ant-node", @@ -892,8 +892,8 @@ dependencies = [ [[package]] name = "ant-node" -version = "0.14.1" -source = "git+https://github.com/grumbach/ant-node?branch=adr-0003-commitment-bound-pricing#195981f53a92f1a6522f8bf96c903c2259c50a09" +version = "0.14.2" +source = "git+https://github.com/grumbach/ant-node?branch=adr-0003-commitment-bound-pricing#ff7a547b627f2a18e630802feb02adeedb34ec1c" dependencies = [ "ant-protocol", "blake3", @@ -941,8 +941,8 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "2.2.1" -source = "git+https://github.com/WithAutonomi/ant-protocol?branch=adr-0003-commitment-bound-pricing#140adea15c4f7e3594d13a017a0770677171322b" +version = "2.3.0" +source = "git+https://github.com/WithAutonomi/ant-protocol?branch=adr-0003-commitment-bound-pricing#3a4eb638abbbec2ec88823f14bfe66e11d70d9b5" dependencies = [ "blake3", "bytes", @@ -2527,8 +2527,8 @@ dependencies = [ [[package]] name = "evmlib" -version = "0.8.1" -source = "git+https://github.com/WithAutonomi/evmlib?branch=adr-0003-signed-quote-fields#4adfdb5615271f716e4233681fb4cf640a0a34d0" +version = "0.9.0" +source = "git+https://github.com/WithAutonomi/evmlib?branch=adr-0003-signed-quote-fields#a7ac7e2620cb6a50151dd0c619fd75e03b02f05c" dependencies = [ "alloy", "ant-merkle", @@ -5247,9 +5247,9 @@ dependencies = [ [[package]] name = "saorsa-core" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3f44ef51271c8bfc5b5305f73831a5090fde070553e2a96636feae109b4fdc4" +checksum = "5d9c05644c46b8de670aa5c8eea037b90731f1f204e8aa5d8a2f03496d740401" dependencies = [ "anyhow", "async-trait", From 440c8c9f7f19953a7d0978c9bd82c4ade063e1f7 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 3 Jul 2026 18:26:44 +0200 Subject: [PATCH 17/22] test(adr-0004): cover client binding signature/hash/count checks Add negative unit tests for the three resolve-before-pay commitment sub-checks that previously had no coverage (deleting any of them still passed all tests): invalid commitment signature, commitment that does not hash to the quote's pin, and quote count disagreeing with the committed key_count. Each constructs a validly-signed, peer-bound commitment and breaks exactly one field to isolate its check. Rename binding_rejects_garbage_and_wrong_pin_commitment to binding_rejects_unparseable_and_peer_unbound_commitment to reflect what it actually exercises (the 'wrong pin' case failed peer-binding first), and fix the stale 'adr-0003' e2e test payload strings. --- ant-core/src/data/client/quote.rs | 94 +++++++++++++++++++++++++++++-- ant-core/tests/e2e_adr0004.rs | 4 +- 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/ant-core/src/data/client/quote.rs b/ant-core/src/data/client/quote.rs index aa870377..87dc59de 100644 --- a/ant-core/src/data/client/quote.rs +++ b/ant-core/src/data/client/quote.rs @@ -2433,6 +2433,36 @@ mod tests { } } + /// Build a VALIDLY-SIGNED `StorageCommitment` bound to `kp`'s peer id, so a + /// test can pass peer-binding + signature and isolate the `hash == pin` and + /// `count == key_count` sub-checks. Mirrors ant-node's commitment signing: + /// the canonical payload (`root || key_count(LE) || peer_id || pk_len(LE) || + /// pub_key`) signed under `DOMAIN_COMMITMENT`. + fn signed_commitment(kp: &Keypair, root: [u8; 32], key_count: u32) -> StorageCommitment { + use ant_protocol::payment::commitment::DOMAIN_COMMITMENT; + use ant_protocol::pqc::api::{ml_dsa_65, MlDsaSecretKey as ApiSecretKey, MlDsaVariant}; + let peer = compute_address(&kp.pub_key_bytes); + let mut payload = Vec::with_capacity(32 + 4 + 32 + 4 + kp.pub_key_bytes.len()); + payload.extend_from_slice(&root); + payload.extend_from_slice(&key_count.to_le_bytes()); + payload.extend_from_slice(&peer); + payload.extend_from_slice(&(kp.pub_key_bytes.len() as u32).to_le_bytes()); + payload.extend_from_slice(&kp.pub_key_bytes); + let sk = ApiSecretKey::from_bytes(MlDsaVariant::MlDsa65, &kp.secret_key_bytes) + .expect("api secret key"); + let signature = ml_dsa_65() + .sign_with_context(&sk, &payload, DOMAIN_COMMITMENT) + .expect("sign commitment") + .to_bytes(); + StorageCommitment { + root, + key_count, + sender_peer_id: peer, + sender_public_key: kp.pub_key_bytes.clone(), + signature, + } + } + #[test] fn binding_baseline_ok_only_at_baseline_price() { // (0, None) priced at calculate_price(0) is the valid baseline. @@ -2485,7 +2515,7 @@ mod tests { } #[test] - fn binding_rejects_garbage_and_wrong_pin_commitment() { + fn binding_rejects_unparseable_and_peer_unbound_commitment() { // A bound quote whose shipped commitment is garbage (doesn't even // deserialize) is rejected — the client never pays an unresolvable pin. let q = quote_with_binding(500, Some([9u8; 32]), calculate_price(500)); @@ -2494,10 +2524,11 @@ mod tests { "an unparseable commitment must be rejected before payment" ); - // A well-formed-but-wrong commitment (valid StorageCommitment bytes that - // do NOT hash to the quote's pin / aren't bound to the peer) is also - // rejected. We serialize a real StorageCommitment shape with mismatched - // fields; it fails peer-binding first, then would fail hash==pin. + // A well-formed StorageCommitment that is NOT bound to the quoting peer + // (its sender_peer_id / pubkey don't derive the peer id) is rejected at + // the peer-binding check. The signature / hash==pin / count==key_count + // sub-checks are covered by the dedicated tests below, which pass + // peer-binding first so each isolates exactly one sub-check. let bogus = StorageCommitment { root: [1u8; 32], key_count: 500, @@ -2512,6 +2543,59 @@ mod tests { ); } + #[test] + fn binding_rejects_commitment_with_invalid_signature() { + // Correctly-bound commitment (passes peer-binding) but with a corrupted + // signature: must be rejected at the signature check. Deleting that check + // would let a peer attest any (root, key_count) without holding the key. + let kp = gen_keypair(); + let mut commitment = signed_commitment(&kp, [6u8; 32], 500); + commitment.signature[0] ^= 0xFF; // still 3293 bytes, no longer valid + // Pin the (corrupted) commitment so the hash==pin check would pass; the + // only thing wrong is the signature, isolating that sub-check. + let pin = commitment_hash(&commitment).expect("hash"); + let blob = rmp_serde::to_vec(&commitment).expect("serialize commitment"); + let q = quote_with_binding(500, Some(pin), calculate_price(500)); + let res = quote_commitment_binding_is_valid(&kp.peer_id, &q, &Some(blob)); + let err = res.expect_err("commitment with an invalid signature must be rejected"); + assert!(err.contains("signature"), "should fail at the signature check: {err}"); + } + + #[test] + fn binding_rejects_commitment_that_does_not_hash_to_pin() { + // Validly-signed, correctly-bound commitment, but the quote pins a + // DIFFERENT hash: must be rejected. Deleting the hash==pin check would + // let a peer ship any commitment it holds for a pin it doesn't back. + let kp = gen_keypair(); + let commitment = signed_commitment(&kp, [5u8; 32], 500); + let wrong_pin = [0xAB; 32]; + assert_ne!(commitment_hash(&commitment), Some(wrong_pin)); + let blob = rmp_serde::to_vec(&commitment).expect("serialize commitment"); + let q = quote_with_binding(500, Some(wrong_pin), calculate_price(500)); + let res = quote_commitment_binding_is_valid(&kp.peer_id, &q, &Some(blob)); + let err = res.expect_err("commitment that does not hash to the pin must be rejected"); + assert!(err.contains("hash"), "should fail at the hash==pin check: {err}"); + } + + #[test] + fn binding_rejects_count_disagreeing_with_commitment() { + // Validly-signed, correctly-bound, correctly-pinned commitment attesting + // key_count=400, but the quote claims 500 (priced on-curve for 500): + // must be rejected. Deleting the count==key_count check would let a peer + // price against an inflated count while committing to fewer keys. + let kp = gen_keypair(); + let commitment = signed_commitment(&kp, [7u8; 32], 400); + let pin = commitment_hash(&commitment).expect("hash"); + let blob = rmp_serde::to_vec(&commitment).expect("serialize commitment"); + let q = quote_with_binding(500, Some(pin), calculate_price(500)); + let res = quote_commitment_binding_is_valid(&kp.peer_id, &q, &Some(blob)); + let err = res.expect_err("a quote count disagreeing with the commitment must be rejected"); + assert!( + err.contains("key_count") || err.contains("attests"), + "should fail at the count==key_count check: {err}" + ); + } + #[test] fn binding_rejects_oversized_commitment_before_parsing() { // A bound quote shipping a blob larger than the sidecar cap is rejected diff --git a/ant-core/tests/e2e_adr0004.rs b/ant-core/tests/e2e_adr0004.rs index 43121e1b..d019c614 100644 --- a/ant-core/tests/e2e_adr0004.rs +++ b/ant-core/tests/e2e_adr0004.rs @@ -45,7 +45,7 @@ async fn adr0004_bound_quotes_are_shipped_priced_and_resolve() { let client = Client::from_node(Arc::clone(&node), test_client_config()) .with_wallet(testnet.wallet().clone()); - let content = Bytes::from("adr-0003 bound-quote payload"); + let content = Bytes::from("adr-0004 bound-quote payload"); let address = compute_address(&content); // Collect quotes directly so we can inspect the ADR-0004 binding the client @@ -147,7 +147,7 @@ async fn adr0004_baseline_quotes_still_work() { let client = Client::from_node(Arc::clone(&node), test_client_config()) .with_wallet(testnet.wallet().clone()); - let content = Bytes::from("adr-0003 baseline payload"); + let content = Bytes::from("adr-0004 baseline payload"); let address = compute_address(&content); let quotes = client From 0414f39a38f91b114bb0507baabfba981818f8f9 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 3 Jul 2026 18:47:41 +0200 Subject: [PATCH 18/22] chore(deps): regenerate Cargo.lock after rebase onto main (0.3.0) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index faf38033..f4f50fa2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -815,7 +815,7 @@ dependencies = [ [[package]] name = "ant-cli" -version = "0.2.9" +version = "0.2.10" dependencies = [ "ant-core", "anyhow", @@ -835,7 +835,7 @@ dependencies = [ [[package]] name = "ant-core" -version = "0.2.9" +version = "0.3.0" dependencies = [ "alloy", "ant-node", From 325e821a809e0157e461200875263da1bfa4a85d Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 3 Jul 2026 18:56:15 +0200 Subject: [PATCH 19/22] style: rustfmt the ADR-0004 binding tests --- ant-core/src/data/client/quote.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ant-core/src/data/client/quote.rs b/ant-core/src/data/client/quote.rs index 87dc59de..3143d71d 100644 --- a/ant-core/src/data/client/quote.rs +++ b/ant-core/src/data/client/quote.rs @@ -2551,14 +2551,17 @@ mod tests { let kp = gen_keypair(); let mut commitment = signed_commitment(&kp, [6u8; 32], 500); commitment.signature[0] ^= 0xFF; // still 3293 bytes, no longer valid - // Pin the (corrupted) commitment so the hash==pin check would pass; the - // only thing wrong is the signature, isolating that sub-check. + // Pin the (corrupted) commitment so the hash==pin check would pass; the + // only thing wrong is the signature, isolating that sub-check. let pin = commitment_hash(&commitment).expect("hash"); let blob = rmp_serde::to_vec(&commitment).expect("serialize commitment"); let q = quote_with_binding(500, Some(pin), calculate_price(500)); let res = quote_commitment_binding_is_valid(&kp.peer_id, &q, &Some(blob)); let err = res.expect_err("commitment with an invalid signature must be rejected"); - assert!(err.contains("signature"), "should fail at the signature check: {err}"); + assert!( + err.contains("signature"), + "should fail at the signature check: {err}" + ); } #[test] @@ -2574,7 +2577,10 @@ mod tests { let q = quote_with_binding(500, Some(wrong_pin), calculate_price(500)); let res = quote_commitment_binding_is_valid(&kp.peer_id, &q, &Some(blob)); let err = res.expect_err("commitment that does not hash to the pin must be rejected"); - assert!(err.contains("hash"), "should fail at the hash==pin check: {err}"); + assert!( + err.contains("hash"), + "should fail at the hash==pin check: {err}" + ); } #[test] From 556256eb28ca27de17ee5bb345d8b459d3ae3671 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 7 Jul 2026 09:53:28 +0900 Subject: [PATCH 20/22] fix(client): drop commitment sidecars from per-chunk merkle proofs Once nodes rotate their first storage commitment, every merkle candidate is bound and finalize_merkle_batch copied all 16 winner-pool commitment sidecars into EVERY per-chunk proof, growing it from ~128 KB to ~342 KB, past the storer's 256 KB payment-proof cap. Every merkle chunk PUT was rejected AFTER the on-chain payment, so forced-merkle uploads burned their payment and then failed every store attempt (DEV-01, 2026-07-06 19:38 UTC: both forced-merkle clients flipped to 100% failure the moment 855 nodes rotated their first commitment; single-node payments were unaffected). The sidecars are not needed in the bundle: the client fully resolves every candidate's shipped commitment before paying, and the storer's cross-check is best-effort with a gossip-cache / GetCommitmentByPin fallback. Drop the forwarding and the dead plumbing that threaded the sidecar map from pool collection into finalization. Receipts cached by pre-fix clients still carry the fat proofs, so a resume would replay the rejection: strip sidecars from cached merkle receipts on load, with an atomic (create_new tmp + fsync + rename) write-back that can never truncate the only copy of a paid receipt. New coverage: - e2e: forced-merkle upload against 35 committed nodes; failed with the exact production signature before this fix, passes after, against the unchanged 256 KB storer cap - unit: finalize_merkle_batch ships no sidecars (bound candidates) - unit: cached receipt strip + idempotency + non-merkle passthrough - unit: atomic overwrite lands and leaves no tmp sibling --- ant-core/src/data/client/cached_merkle.rs | 203 +++++++++++++++++++++- ant-core/src/data/client/merkle.rs | 129 +++++++------- ant-core/tests/e2e_adr0004.rs | 44 ++++- 3 files changed, 308 insertions(+), 68 deletions(-) diff --git a/ant-core/src/data/client/cached_merkle.rs b/ant-core/src/data/client/cached_merkle.rs index 78b83beb..358711a3 100644 --- a/ant-core/src/data/client/cached_merkle.rs +++ b/ant-core/src/data/client/cached_merkle.rs @@ -49,7 +49,8 @@ use crate::config; use crate::data::client::merkle::MerkleBatchPaymentResult; use crate::error::Result; -use std::fs::{self, DirEntry, File}; +use ant_protocol::payment::{deserialize_merkle_proof, serialize_merkle_proof}; +use std::fs::{self, DirEntry, File, OpenOptions}; use std::hash::{Hash, Hasher}; use std::io::{BufReader, BufWriter}; use std::path::{Path, PathBuf}; @@ -283,11 +284,112 @@ fn is_expired_filename(name: &str) -> bool { fn read_receipt(path: &Path) -> Result { let handle = File::open(path)?; - let receipt: MerkleBatchPaymentResult = rmp_serde::decode::from_read(BufReader::new(handle)) - .map_err(|e| crate::error::Error::Io(std::io::Error::other(e.to_string())))?; + let mut receipt: MerkleBatchPaymentResult = + rmp_serde::decode::from_read(BufReader::new(handle)) + .map_err(|e| crate::error::Error::Io(std::io::Error::other(e.to_string())))?; + + if strip_commitment_sidecars(&mut receipt) { + info!( + "Stripped legacy commitment sidecars from cached merkle receipt at {}", + path.display() + ); + // Best-effort write-back so the strip happens once; a failure here + // only means we re-strip on the next load. + if let Err(e) = overwrite_receipt(path, &receipt) { + warn!( + "Failed to persist slimmed merkle receipt at {}: {e}", + path.display() + ); + } + } + Ok(receipt) } +/// Strip ADR-0004 commitment sidecars from every proof in a cached receipt. +/// +/// Receipts saved by clients built before sidecars were dropped from the +/// per-chunk merkle proofs carry all 16 winner-pool sidecars in EVERY proof +/// (~214 KB per proof), which pushed the proof past the storer's +/// payment-proof size cap — resuming with them would replay the exact +/// failure the slim proofs fixed. Stripping is always safe: the pool hash +/// and address branch stay exactly as paid on-chain, and storers resolve +/// commitment pins from gossip or a `GetCommitmentByPin` fetch. Returns +/// whether anything was stripped. +fn strip_commitment_sidecars(receipt: &mut MerkleBatchPaymentResult) -> bool { + let mut stripped = false; + for proof_bytes in receipt.proofs.values_mut() { + // Non-merkle or unreadable proof bytes are left untouched; the + // storer remains the judge of those. + let Ok(mut proof) = deserialize_merkle_proof(proof_bytes) else { + continue; + }; + if proof.commitment_sidecars.is_empty() { + continue; + } + proof.commitment_sidecars.clear(); + match serialize_merkle_proof(&proof) { + Ok(slim) => { + *proof_bytes = slim; + stripped = true; + } + Err(e) => warn!("Failed to re-serialize slimmed cached merkle proof: {e}"), + } + } + stripped +} + +/// Overwrite a cached receipt via `tmp + fsync + rename` (same canonical +/// path), mirroring `cached_single::write_receipt_atomic`: an interrupted +/// write must never truncate the only copy of a paid receipt — losing it +/// forces the user to re-pay. The tmp name carries pid + nanos and is opened +/// with `create_new`, so concurrent migrations (threads or processes) can +/// never share a tmp inode — a residual name collision fails this best-effort +/// write instead of corrupting it, and the next load simply re-strips. A +/// leftover tmp from a crash is harmless (the canonical stays intact until +/// rename) and ages out with the same `_` filename prefix. +fn overwrite_receipt(path: &Path, receipt: &MerkleBatchPaymentResult) -> Result<()> { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let pid = std::process::id(); + let tmp_path = path.with_extension(format!("{pid}-{nanos}.tmp")); + { + let handle = OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp_path)?; + let mut writer = BufWriter::new(handle); + if let Err(e) = rmp_serde::encode::write(&mut writer, receipt) { + let _ = fs::remove_file(&tmp_path); + return Err(crate::error::Error::Io(std::io::Error::other( + e.to_string(), + ))); + } + // `into_inner` flushes; a swallowed flush error here would defeat + // the atomicity, so surface it. + let handle = match writer.into_inner() { + Ok(handle) => handle, + Err(e) => { + let _ = fs::remove_file(&tmp_path); + return Err(crate::error::Error::Io(std::io::Error::other(format!( + "BufWriter flush failed: {e}" + )))); + } + }; + if let Err(e) = handle.sync_all() { + let _ = fs::remove_file(&tmp_path); + return Err(e.into()); + } + } + if let Err(e) = fs::rename(&tmp_path, path) { + let _ = fs::remove_file(&tmp_path); + return Err(e.into()); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -305,6 +407,101 @@ mod tests { } } + /// A serialized merkle proof carrying legacy commitment sidecars, as saved + /// by clients built before sidecars were dropped from per-chunk proofs. + fn fat_merkle_proof_bytes(ts: u64) -> Vec { + use ant_protocol::evm::{ + Amount, MerklePaymentCandidateNode, MerklePaymentCandidatePool, MerklePaymentProof, + MerkleTree, RewardsAddress, CANDIDATES_PER_POOL, + }; + use xor_name::XorName; + + let xornames: Vec = (0..4u8).map(|i| XorName([i; 32])).collect(); + let tree = MerkleTree::from_xornames(xornames.clone()).unwrap(); + let midpoint = tree.reward_candidates(ts).unwrap().remove(0); + let candidate_nodes: [MerklePaymentCandidateNode; CANDIDATES_PER_POOL] = + std::array::from_fn(|i| MerklePaymentCandidateNode { + pub_key: vec![i as u8; 32], + price: Amount::from(1024u64), + reward_address: RewardsAddress::new([i as u8; 20]), + merkle_payment_timestamp: ts, + signature: vec![i as u8; 64], + committed_key_count: 9_000, + commitment_pin: Some([7u8; 32]), + }); + let pool = MerklePaymentCandidatePool { + midpoint_proof: midpoint, + candidate_nodes, + }; + let address_proof = tree.generate_address_proof(0, xornames[0]).unwrap(); + let mut proof = MerklePaymentProof::new(xornames[0], address_proof, pool); + proof.commitment_sidecars = vec![vec![0xAB; 5_000]; CANDIDATES_PER_POOL]; + serialize_merkle_proof(&proof).unwrap() + } + + /// DEV-01 recovery: a receipt cached by a pre-fix client carries proofs + /// with all 16 commitment sidecars (~342 KB each on the wire) — resuming + /// with them would replay the storer's size rejection. Loading must strip + /// the sidecars while leaving the paid pool and address branch intact. + #[test] + fn strip_removes_legacy_sidecars_and_is_idempotent() { + let ts = 1_000_000; + let fat = fat_merkle_proof_bytes(ts); + let mut proofs: HashMap<[u8; 32], Vec> = HashMap::new(); + proofs.insert([0u8; 32], fat.clone()); + // A non-merkle blob must pass through untouched. + proofs.insert([1u8; 32], vec![1, 2, 3]); + let mut receipt = MerkleBatchPaymentResult { + proofs, + chunk_count: 2, + storage_cost_atto: "0".to_string(), + gas_cost_wei: 0, + merkle_payment_timestamp: ts, + }; + + assert!(strip_commitment_sidecars(&mut receipt)); + + let slim = receipt.proofs.get(&[0u8; 32]).unwrap(); + assert!(slim.len() < fat.len(), "stripped proof must shrink"); + let proof = deserialize_merkle_proof(slim).unwrap(); + assert!(proof.commitment_sidecars.is_empty()); + assert_eq!(receipt.proofs.get(&[1u8; 32]).unwrap(), &vec![1, 2, 3]); + + // Second pass finds nothing left to strip. + assert!(!strip_commitment_sidecars(&mut receipt)); + } + + /// The strip write-back must replace the canonical receipt atomically: + /// new content lands, and no `.tmp` sibling survives a successful write. + #[test] + fn overwrite_receipt_is_atomic_and_leaves_no_tmp() -> Result<()> { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let dir = std::env::temp_dir().join(format!("anselme-merkle-overwrite-test-{nanos}")); + fs::create_dir_all(&dir)?; + let path = dir.join("123_abcd"); + fs::write(&path, b"pre-fix receipt bytes")?; + + overwrite_receipt(&path, &dummy_receipt(42))?; + + let reloaded = read_receipt(&path)?; + assert_eq!(reloaded.merkle_payment_timestamp, 42); + let leftover_tmps = fs::read_dir(&dir)? + .flatten() + .filter(|e| { + e.path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("tmp")) + }) + .count(); + assert_eq!(leftover_tmps, 0, "no tmp sibling may survive"); + + fs::remove_dir_all(&dir).ok(); + Ok(()) + } + #[test] fn file_hash_key_is_stable() { let a = file_hash_key("/tmp/some/file.bin"); diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index 091a3f6d..af0e44ba 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -163,10 +163,6 @@ pub struct PreparedMerkleBatch { tree: MerkleTree, /// Internal: chunk addresses in order. addresses: Vec<[u8; 32]>, - /// ADR-0004: validated commitment sidecars keyed by `(peer, pin)`, collected - /// during candidate validation. At proof time the winner pool's candidates - /// forward theirs in `MerklePaymentProof.commitment_sidecars`. - commitment_sidecars: HashMap<(PeerId, [u8; 32]), Vec>, } /// Result of checking a merkle upload batch before payment. @@ -528,9 +524,11 @@ impl Client { ); // 3. Collect candidate pools from the network (all pools in parallel). - // Each candidate's ADR-0004 binding is verified during collection and - // its commitment sidecar captured (keyed by peer, pin). - let (candidate_pools, commitment_sidecars) = self + // Each candidate's ADR-0004 binding is fully verified during + // collection (shape, cap, exact price, commitment resolution); the + // sidecars themselves are consumed by that validation and NOT + // forwarded in the PUT bundles (see `finalize_merkle_batch`). + let candidate_pools = self .build_candidate_pools( &midpoint_proofs, data_type, @@ -552,7 +550,6 @@ impl Client { candidate_pools, tree, addresses: addresses.to_vec(), - commitment_sidecars, }) } @@ -666,17 +663,14 @@ impl Client { data_type: u32, data_size: u64, merkle_payment_timestamp: u64, - ) -> Result<( - Vec, - HashMap<(PeerId, [u8; 32]), Vec>, - )> { + ) -> Result> { let mut pool_futures = FuturesUnordered::new(); for midpoint_proof in midpoint_proofs { let pool_address = midpoint_proof.address(); let mp = midpoint_proof.clone(); pool_futures.push(async move { - let (candidate_nodes, sidecars) = self + let candidate_nodes = self .get_merkle_candidate_pool( &pool_address.0, data_type, @@ -684,28 +678,19 @@ impl Client { merkle_payment_timestamp, ) .await?; - Ok::<_, Error>(( - MerklePaymentCandidatePool { - midpoint_proof: mp, - candidate_nodes, - }, - sidecars, - )) + Ok::<_, Error>(MerklePaymentCandidatePool { + midpoint_proof: mp, + candidate_nodes, + }) }); } let mut pools = Vec::with_capacity(midpoint_proofs.len()); - // ADR-0004: merged map of every validated candidate's commitment sidecar, - // keyed by (peer, pin). At proof time the winner pool's candidates look - // up their sidecars here to forward in the merkle PUT bundle. - let mut sidecars: HashMap<(PeerId, [u8; 32]), Vec> = HashMap::new(); while let Some(result) = pool_futures.next().await { - let (pool, pool_sidecars) = result?; - pools.push(pool); - sidecars.extend(pool_sidecars); + pools.push(result?); } - Ok((pools, sidecars)) + Ok(pools) } /// Collect `CANDIDATES_PER_POOL` (16) merkle candidate quotes from the network. @@ -716,10 +701,7 @@ impl Client { data_type: u32, data_size: u64, merkle_payment_timestamp: u64, - ) -> Result<( - [MerklePaymentCandidateNode; CANDIDATES_PER_POOL], - HashMap<(PeerId, [u8; 32]), Vec>, - )> { + ) -> Result<[MerklePaymentCandidateNode; CANDIDATES_PER_POOL]> { let node = self.network().node(); let timeout = Duration::from_secs(self.config().quote_timeout_secs); @@ -853,13 +835,9 @@ impl Client { >, target_address: &[u8; 32], merkle_payment_timestamp: u64, - ) -> Result<( - [MerklePaymentCandidateNode; CANDIDATES_PER_POOL], - HashMap<(PeerId, [u8; 32]), Vec>, - )> { + ) -> Result<[MerklePaymentCandidateNode; CANDIDATES_PER_POOL]> { let mut valid: Vec<(PeerId, MerklePaymentCandidateNode)> = Vec::new(); let mut failures: Vec = Vec::new(); - let mut sidecars: HashMap<(PeerId, [u8; 32]), Vec> = HashMap::new(); while let Some((peer_id, result)) = futures.next().await { match result { @@ -875,10 +853,9 @@ impl Client { continue; } // The candidate's identity is `BLAKE3(candidate.pub_key)` — - // this is what the storer derives (verifier.rs) and what proof - // finalization keys the sidecar by. Require it to equal the - // network responder so a two-identity operator cannot answer - // as B while shipping A's commitment. + // this is what the storer derives (verifier.rs). Require it + // to equal the network responder so a two-identity operator + // cannot answer as B while shipping A's commitment. let candidate_peer = PeerId::from_bytes(compute_address(&candidate.pub_key)); if candidate_peer != peer_id { warn!( @@ -892,7 +869,10 @@ impl Client { // the single-node path — a candidate priced off its committed // count, or shipping an unresolvable/forged commitment, is // dropped before it can enter a pool the client pays. Checked - // against the CANDIDATE peer (the one the storer audits). + // against the CANDIDATE peer (the one the storer audits). The + // shipped commitment is consumed here (resolution only) and + // not forwarded in the PUT bundles (see + // `finalize_merkle_batch`). if let Err(detail) = merkle_candidate_binding_is_valid(&candidate_peer, &candidate, &commitment) { @@ -900,11 +880,6 @@ impl Client { failures.push(format!("{peer_id}: bad commitment binding ({detail})")); continue; } - // Key the sidecar by the CANDIDATE peer (== BLAKE3(pub_key)) so - // proof finalization, which derives the same key, finds it. - if let (Some(pin), Some(blob)) = (candidate.commitment_pin, commitment) { - sidecars.insert((candidate_peer, pin), blob); - } valid.push((candidate_peer, candidate)); } Err(e) => { @@ -935,7 +910,7 @@ impl Client { candidates.try_into().map_err(|_| { Error::Payment("Failed to convert candidates to fixed array".to_string()) })?; - Ok((array, sidecars)) + Ok(array) } /// Upload chunks using pre-computed merkle proofs from a batch payment. @@ -1446,18 +1421,14 @@ pub fn finalize_merkle_batch( )) })?; - // ADR-0004: collect the winner pool's candidate commitment sidecars (those - // that were bound + validated during collection), to forward in each proof - // so the storer can cross-check synchronously. Built once for the pool. - let winner_sidecars: Vec> = winner_pool - .candidate_nodes - .iter() - .filter_map(|c| { - let pin = c.commitment_pin?; - let peer = PeerId::from_bytes(compute_address(&c.pub_key)); - prepared.commitment_sidecars.get(&(peer, pin)).cloned() - }) - .collect(); + // ADR-0004: commitment sidecars are deliberately NOT forwarded in the + // per-chunk proofs. Sixteen sidecars are ~214 KB serialized, and copying + // them into every chunk's bundle pushed the proof past the storer's + // payment-proof size cap, rejecting every merkle PUT once nodes carried + // live commitments. The client still fully resolves every candidate's + // commitment before paying (during pool collection); the storer's + // cross-check is best-effort and resolves pins from its gossip cache or a + // `GetCommitmentByPin` fetch when no sidecar is shipped. // Generate proofs for each chunk info!("Generating merkle proofs for {chunk_count} chunks"); @@ -1473,9 +1444,7 @@ pub fn finalize_merkle_batch( )) })?; - let mut merkle_proof = - MerklePaymentProof::new(*xorname, address_proof, winner_pool.clone()); - merkle_proof.commitment_sidecars = winner_sidecars.clone(); + let merkle_proof = MerklePaymentProof::new(*xorname, address_proof, winner_pool.clone()); let tagged_bytes = serialize_merkle_proof(&merkle_proof) .map_err(|e| Error::Serialization(format!("Failed to serialize merkle proof: {e}")))?; @@ -1866,7 +1835,6 @@ mod tests { candidate_pools, tree, addresses: addrs, - commitment_sidecars: HashMap::new(), } } @@ -1891,6 +1859,39 @@ mod tests { } } + /// DEV-01 regression: per-chunk merkle proofs must never ship commitment + /// sidecars — all 16 winner-pool sidecars (~214 KB serialized) copied into + /// every chunk's proof pushed it past the storer's payment-proof size cap, + /// rejecting every merkle PUT once nodes carried live commitments. The + /// e2e (`adr0004_merkle_upload_against_bound_candidates`) proves the flow + /// end-to-end; this pins the wire invariant directly so it cannot slip + /// back in behind a raised node-side cap. + #[test] + fn test_finalize_merkle_batch_ships_no_commitment_sidecars() { + use ant_protocol::payment::deserialize_merkle_proof; + + let mut prepared = make_prepared_merkle_batch(4); + // Bind every candidate to a pin, mirroring a network where all nodes + // carry live commitments (the DEV-01 trigger state). + for pool in &mut prepared.candidate_pools { + for candidate in &mut pool.candidate_nodes { + candidate.committed_key_count = 9_000; + candidate.commitment_pin = Some([7u8; 32]); + } + } + let winner_hash = prepared.candidate_pools[0].hash(); + + let batch = finalize_merkle_batch(prepared, winner_hash).unwrap(); + assert_eq!(batch.proofs.len(), 4); + for proof_bytes in batch.proofs.values() { + let proof = deserialize_merkle_proof(proof_bytes).unwrap(); + assert!( + proof.commitment_sidecars.is_empty(), + "per-chunk merkle proofs must not ship commitment sidecars" + ); + } + } + #[test] fn test_finalize_merkle_batch_with_invalid_winner() { let prepared = make_prepared_merkle_batch(4); diff --git a/ant-core/tests/e2e_adr0004.rs b/ant-core/tests/e2e_adr0004.rs index d019c614..9a49dace 100644 --- a/ant-core/tests/e2e_adr0004.rs +++ b/ant-core/tests/e2e_adr0004.rs @@ -22,13 +22,15 @@ mod support; -use ant_core::data::{compute_address, Client}; +use ant_core::data::client::merkle::PaymentMode; +use ant_core::data::{compute_address, Client, ClientConfig}; use ant_protocol::payment::calculate_price; use ant_protocol::payment::commitment::{ commitment_hash, verify_commitment_signature, StorageCommitment, }; use bytes::Bytes; use serial_test::serial; +use std::io::Write; use std::sync::Arc; use support::{test_client_config, MiniTestnet, DEFAULT_NODE_COUNT}; @@ -184,3 +186,43 @@ async fn adr0004_baseline_quotes_still_work() { drop(client); testnet.teardown().await; } + +/// DEV-01 regression: a FORCED-MERKLE upload against a network where every +/// node carries a live storage commitment (bound candidates). On DEV-01 this +/// exact combination started failing the moment nodes rotated their first +/// commitment (2026-07-06 19:38 UTC) while single-node payments kept working. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr0004_merkle_upload_against_bound_candidates() { + // 35 nodes: merkle pools need 16 valid candidates per pool. + let testnet = MiniTestnet::start_with_commitments(35, COMMITTED_KEYS).await; + let node = testnet.node(5).expect("node 5 exists"); + let config = ClientConfig { + quote_timeout_secs: 120, + store_timeout_secs: 120, + close_group_size: 20, + ..Default::default() + }; + let client = Client::from_node(Arc::clone(&node), config).with_wallet(testnet.wallet().clone()); + + // 500KB file — self-encryption produces 3+ chunks (>=2 needed for merkle). + let data: Vec = (0u8..=255).cycle().take(500_000).collect(); + let mut input_file = tempfile::NamedTempFile::new().expect("create temp file"); + input_file.write_all(&data).expect("write temp file"); + input_file.flush().expect("flush temp file"); + + let result = client + .file_upload_with_mode(input_file.path(), PaymentMode::Merkle) + .await + .expect("merkle upload against bound candidates must succeed"); + + assert_eq!( + result.payment_mode_used, + PaymentMode::Merkle, + "payment_mode_used must be Merkle" + ); + assert!(result.chunks_stored >= 3, "chunks must be stored"); + + drop(client); + testnet.teardown().await; +} From 4e3aa7b6e551d77a69587b3c3b84d5270b3d1658 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 7 Jul 2026 10:07:15 +0900 Subject: [PATCH 21/22] chore(deps): bump crossbeam-epoch 0.9.18 -> 0.9.20 (RUSTSEC-2026-0204) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4f50fa2..18e0c744 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2008,9 +2008,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] From 22cf9fb104881c585bad17de27eed9cb66fe550c Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 9 Jul 2026 14:15:43 +0900 Subject: [PATCH 22/22] Add ADR-0005 client-side earned reward eligibility gate The client decides who is payable. This adds the eligibility gate at quote collection (payee selection and payment verification untouched): a quoter is payable when more than half the other responders who know it vouch for a clean audited week at the quoted size. - eligibility: aggregate the quorum of signed audit reports into a two-tier decision. Tier 1 (size) needs a vouched clean week AT the quoted size; tier 2 (dues) relaxes size to keep the network paying under fast growth, and is safe because ADR-0004 independently forces a current-size proof at payment. Both tiers still exclude fresh identities (no dues) and caught cheaters (fenced/convicted). The eligibility map is keyed by (peer, quoted size) so one peer's small quote can never leak size-tier admission onto its larger quote. If too few are eligible, it prefers the ones with the most dues, then falls back to today's rules; the network never stalls. - merkle / quote: carry the client report_nonce on quote requests, collect the returned audit reports, and apply the gate at pool / close-group composition. - tests: unit coverage for the bar, both tiers, and the exclusions, plus an end-to-end suite (mixed pool under enforcement, fenced/unproven substitution, and fast-growth dues fallback). - Cargo: temporarily pin ant-protocol and ant-node to the ADR-0005 PR branches carrying the wire types and node tally; revert to published bumps once released. Based on the ADR-0004 client work in #126. --- Cargo.lock | 6 +- Cargo.toml | 20 +- ant-core/Cargo.toml | 4 +- ant-core/examples/bench-quoting.rs | 2 + ant-core/src/data/client/eligibility.rs | 800 ++++++++++++++++++++++++ ant-core/src/data/client/merkle.rs | 85 ++- ant-core/src/data/client/mod.rs | 1 + ant-core/src/data/client/quote.rs | 310 ++++++++- ant-core/tests/e2e_adr0005.rs | 529 ++++++++++++++++ ant-core/tests/support/mod.rs | 165 ++++- 10 files changed, 1872 insertions(+), 50 deletions(-) create mode 100644 ant-core/src/data/client/eligibility.rs create mode 100644 ant-core/tests/e2e_adr0005.rs diff --git a/Cargo.lock b/Cargo.lock index 18e0c744..17c4c30e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -892,8 +892,8 @@ dependencies = [ [[package]] name = "ant-node" -version = "0.14.2" -source = "git+https://github.com/grumbach/ant-node?branch=adr-0003-commitment-bound-pricing#ff7a547b627f2a18e630802feb02adeedb34ec1c" +version = "0.14.3" +source = "git+https://github.com/grumbach/ant-node?branch=adr-0005-earned-reward-eligibility#ec8b12be06f3049ad1015a52044a28792033a7b9" dependencies = [ "ant-protocol", "blake3", @@ -942,7 +942,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "2.3.0" -source = "git+https://github.com/WithAutonomi/ant-protocol?branch=adr-0003-commitment-bound-pricing#3a4eb638abbbec2ec88823f14bfe66e11d70d9b5" +source = "git+https://github.com/grumbach/ant-protocol?branch=adr-0005-earned-reward-eligibility#246ad516c87eb0531127abff86a1258d842052b8" dependencies = [ "blake3", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 745a1102..a87bf330 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,20 +2,22 @@ members = ["ant-core", "ant-cli"] resolver = "2" -# ─── TEMPORARY: STRIP BEFORE MERGE — ADR-0004 coordinated cutover ─── -# ant-protocol/evmlib's ADR-0004 commitment-bound-quote types (the signed -# `committed_key_count`/`commitment_pin` fields, the commitment sidecar, and -# `BadQuoteCommitment`) are not yet published to crates.io, so this workspace -# cannot build against the published `ant-protocol`/`evmlib`. These git patches -# point those deps at their PR branches so CI can build the coordinated change. +# ─── TEMPORARY: STRIP BEFORE MERGE — ADR-0004/0005 coordinated cutover ─── +# The ADR-0004 commitment-bound-quote types (signed `committed_key_count`/ +# `commitment_pin`, the commitment sidecar) and the ADR-0005 audit-report wire +# types are not yet published to crates.io, so this workspace cannot build +# against the published `ant-protocol`/`evmlib`. These git patches point those +# deps at their PR branches so CI can build the coordinated change. # -# AT RELEASE (once the ADR-0004 versions are published, in order +# AT RELEASE (once the ADR-0004/0005 versions are published, in order # evmlib → ant-protocol): # 1. delete this entire [patch.crates-io] block, AND # 2. bump the `ant-protocol` pin in ant-core/Cargo.toml to the published # version (evmlib is re-exported through ant-protocol, no direct pin): -# ant-protocol = "2.3.0" # <- published ADR-0004 ant-protocol +# ant-protocol = "2.3.0" # <- published ant-protocol # # (which in turn depends on the published evmlib = "0.9.0") [patch.crates-io] evmlib = { git = "https://github.com/WithAutonomi/evmlib", branch = "adr-0003-signed-quote-fields" } -ant-protocol = { git = "https://github.com/WithAutonomi/ant-protocol", branch = "adr-0003-commitment-bound-pricing" } +# ADR-0005 audit-report wire types (github.com/WithAutonomi/ant-protocol/pull/19), +# based on the ADR-0004 protocol branch and pinning the published evmlib 0.9.0. +ant-protocol = { git = "https://github.com/grumbach/ant-protocol", branch = "adr-0005-earned-reward-eligibility" } diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 59ea1ce9..cb6c3bbc 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -65,7 +65,7 @@ sysinfo = { version = "0.32", default-features = false, features = ["system"] } # `ant-protocol` pin above points at a git branch, this ant-node must point at # the matching ant-node branch carrying the same saorsa-core / ant-protocol # lineage rather than a released version. -ant-node = { git = "https://github.com/grumbach/ant-node", branch = "adr-0003-commitment-bound-pricing", optional = true } +ant-node = { git = "https://github.com/grumbach/ant-node", branch = "adr-0005-earned-reward-eligibility", optional = true } tracing-subscriber = { version = "0.3", features = ["env-filter"] } [target.'cfg(unix)'.dependencies] @@ -96,7 +96,7 @@ test-utils = [] # always compile even without the `devnet` feature. Pinned to the same # version as the runtime dep so there is a single ant-node / # saorsa-core version across the whole graph. -ant-node = { git = "https://github.com/grumbach/ant-node", branch = "adr-0003-commitment-bound-pricing", features = ["test-utils"] } +ant-node = { git = "https://github.com/grumbach/ant-node", branch = "adr-0005-earned-reward-eligibility", features = ["test-utils"] } serial_test = "3" anyhow = "1" alloy = { version = "1.6", features = ["node-bindings"] } diff --git a/ant-core/examples/bench-quoting.rs b/ant-core/examples/bench-quoting.rs index abf14865..1b59d8e2 100644 --- a/ant-core/examples/bench-quoting.rs +++ b/ant-core/examples/bench-quoting.rs @@ -356,6 +356,7 @@ async fn bench_normal_once(client: &Client, rep: usize) -> Rep { address, data_size: 1024, data_type: 0, + report_nonce: [0u8; 32], }), }; let bytes = match msg.encode() { @@ -509,6 +510,7 @@ async fn bench_merkle_once(client: &Client, rep: usize, concurrency: usize) -> R data_type: 0, data_size: 1024, merkle_payment_timestamp, + report_nonce: [0u8; 32], }), }; let bytes = match msg.encode() { diff --git a/ant-core/src/data/client/eligibility.rs b/ant-core/src/data/client/eligibility.rs new file mode 100644 index 00000000..cd6aaf3d --- /dev/null +++ b/ant-core/src/data/client/eligibility.rs @@ -0,0 +1,800 @@ +//! ADR-0005 earned reward eligibility — the client-side predicate. +//! +//! Quote responses carry each responder's signed **audit report**: its raw +//! per-peer tally of audit outcomes (passes per day, the largest commitment +//! size that passed, and two row-local markers). This module aggregates a +//! quorum of those reports into the payee-selection decision: a quoter is +//! **eligible** when enough of its neighbours testify it has been audited +//! clean for about a week at the size it is monetizing. +//! +//! Everything here is policy over reported facts — no scores, no weights, no +//! decay. The parameters are client-side and env-tunable, so retuning the +//! gate never needs a fleet release. Reports are consumed at collection time +//! and dropped; nothing here is forwarded to storers. +//! +//! Default mode is **observe-only**: the gate logs what it would do without +//! changing selection. `ADR5_ENFORCE=1` turns enforcement on (used by the +//! local testnet; production flips only once telemetry shows honest nodes +//! reliably qualify). + +use ant_protocol::payment::{AuditReport, AuditReportDay, AuditReportRow}; +use ant_protocol::transport::PeerId; +use std::collections::HashMap; +use tracing::{debug, info, warn}; + +/// Client eligibility policy over reported audit facts (ADR-0005). +/// +/// Day units are the NODE's tally-day buckets (`ADR5_DAY_SECS` on nodes), so +/// time compression on a testnet applies to both sides consistently. +#[derive(Debug, Clone, Copy)] +pub struct EligibilityPolicy { + /// A qualifying observer must report at least this many distinct days + /// that each carry a covering pass (D — the dues, ~a week). + pub min_distinct_days: u16, + /// The most recent qualifying day must be at most this old (the window's + /// head must be covered: a node that stops answering audits drops out + /// within ~a day). + pub max_recency_days: u16, + /// Size coverage slack: a day qualifies only if its largest passed + /// commitment size × this multiplier ≥ the quoted count (grind small, + /// cash in big fails day-coverage; honest growth clears in an audit + /// cycle). + pub size_slack: u32, + /// Days older than this are ignored entirely — the client's own window + /// bound, independent of what a reporter chooses to ship. + pub window_days: u16, + /// Floor of the eligibility bar (ADR-0005 v4): a subject always needs at + /// least this many qualifying vouches, however few reporters know it. + pub quorum_floor: usize, + /// Enforce (change selection) vs observe-only (log would-be decisions). + pub enforce: bool, +} + +impl Default for EligibilityPolicy { + fn default() -> Self { + // NOTE: `window_days` can only NARROW the node's `TALLY_WINDOW_DAYS`, it + // cannot widen it; and there is NO client knob for conviction + // stickiness (node-side `CONVICTION_STICKY_DAYS`). "Retunable without a + // node release" is true for D / recency / slack / floor / enforce ONLY. + Self { + min_distinct_days: env_u64_clamped("ADR5_D_DAYS", 7, 0, u64::from(u16::MAX)) as u16, + max_recency_days: env_u64_clamped("ADR5_RECENCY_DAYS", 1, 0, u64::from(u16::MAX)) + as u16, + size_slack: env_u64_clamped("ADR5_SIZE_SLACK", 2, 1, u64::from(u32::MAX)) as u32, + window_days: env_u64_clamped("ADR5_WINDOW_DAYS", 14, 0, u64::from(u16::MAX)) as u16, + quorum_floor: env_u64_clamped("ADR5_QUORUM", 3, 1, u64::from(u32::MAX)) as usize, + enforce: std::env::var("ADR5_ENFORCE").is_ok_and(|v| v == "1" || v == "true"), + } + } +} + +/// Read `name` as a u64, clamped to `[lo, hi]`, defaulting to `default`. +/// WARNs (not silently) when a set value is non-numeric (falls back to +/// default) or out of range (clamped) — otherwise an operator who typed +/// `ADR5_SIZE_SLACK=0` or a typo would get the opposite of what they meant +/// with no signal. +fn env_u64_clamped(name: &str, default: u64, lo: u64, hi: u64) -> u64 { + let Some(raw) = std::env::var(name).ok() else { + return default.clamp(lo, hi); + }; + let Some(parsed) = raw.parse::().ok() else { + warn!("ADR-0005: {name}={raw:?} is not a number; using default {default}"); + return default.clamp(lo, hi); + }; + let clamped = parsed.clamp(lo, hi); + if clamped != parsed { + warn!("ADR-0005: {name}={parsed} out of range [{lo}, {hi}]; clamped to {clamped}"); + } + clamped +} + +/// Does one report day cover a quote of `quoted_key_count` under `policy`'s +/// size slack? (`max_passed_key_count * slack >= quoted`.) A baseline quote +/// (`quoted == 0`) is trivially covered by any pass. +#[must_use] +fn day_covers_size( + day: &AuditReportDay, + quoted_key_count: u32, + policy: &EligibilityPolicy, +) -> bool { + u64::from(day.max_passed_key_count) * u64::from(policy.size_slack) + >= u64::from(quoted_key_count) +} + +/// Shared history check for one observer's row: unfenced, unconvicted, and +/// enough DISTINCT recent days that satisfy `day_qualifies`. +/// +/// This is the single place the ADR-0005 "≥ D distinct covering days, freshest +/// within recency, inside the window" rule lives. The per-day `day_qualifies` +/// predicate is what makes it size-aware or size-relaxed: +/// - size-eligible ([`row_qualifies`]): `day_covers_size` — the day must +/// also cover the quoted size, exactly as v4. +/// - dues-eligible ([`row_has_dues`], v5 tier 2): `|_| true` — any pass day. +/// +/// Crucially the size check is applied PER DAY, INSIDE the distinct-day and +/// recency computation — it is NOT a separate whole-row predicate. Factoring +/// it out (`has_dues && covers_size_somewhere`) would be WRONG: a row with 7 +/// small passes at ages 0-6 plus one covering pass at age 13 has dues and a +/// covering day, yet must NOT be size-eligible (only one covering day, and it +/// is stale). Keeping size a day predicate preserves v4 semantics exactly. +/// +/// Duplicate `age_days` collapse to one distinct day, so a malformed report +/// cannot multiply its own testimony. Pass counts are not summed — a per-day +/// `passes != 0` guard is the only pass requirement (there is no total-passes +/// threshold). +fn row_meets_history( + row: &AuditReportRow, + policy: &EligibilityPolicy, + day_qualifies: impl Fn(&AuditReportDay) -> bool, +) -> bool { + if row.fenced || row.convicted { + return false; + } + let mut qualifying_ages: std::collections::HashSet = std::collections::HashSet::new(); + for day in &row.days { + if day.passes == 0 || day.age_days > policy.window_days { + continue; + } + if day_qualifies(day) { + qualifying_ages.insert(day.age_days); + } + } + let freshest_age = qualifying_ages.iter().min().copied(); + qualifying_ages.len() >= usize::from(policy.min_distinct_days) + && freshest_age.is_some_and(|age| age <= policy.max_recency_days) +} + +/// Does `row` qualify the subject for a quote **at `quoted_key_count`** (the +/// SIZE-eligible tier)? Unfenced, unconvicted, and ≥ D distinct recent days +/// that each cover the quoted size. Identical semantics to v4. +#[must_use] +pub fn row_qualifies( + row: &AuditReportRow, + quoted_key_count: u32, + policy: &EligibilityPolicy, +) -> bool { + row_meets_history(row, policy, |day| { + day_covers_size(day, quoted_key_count, policy) + }) +} + +/// Does `row` show the subject has done its **dues** (the size-RELAXED tier 2, +/// ADR-0005 v5)? Same history rule as [`row_qualifies`] but with NO size +/// coverage: ≥ D distinct recent pass days at ANY size. A fresh identity (no +/// pass days) and a caught cheater (fenced/convicted) still fail — the +/// fallback drops only the size requirement, never the dues or the catch. +/// +/// Every size-eligible row is dues-eligible: its covering days are a subset of +/// its any-size pass days, so it has ≥ D such days and its freshest any-size +/// day is no older than its freshest covering day. +#[must_use] +pub fn row_has_dues(row: &AuditReportRow, policy: &EligibilityPolicy) -> bool { + row_meets_history(row, policy, |_day| true) +} + +/// Which tier a subject cleared (ADR-0005 v5 two-tier fallback). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Tier { + /// Majority of judges vouch AT THE QUOTED SIZE — full eligibility. + Size, + /// Majority of judges vouch for the DUES (a clean audited week at any + /// size) but too few cover the quoted size — the v5 size-relaxed fallback. + Dues, +} + +/// One subject's eligibility, once it has cleared at least the dues bar. Every +/// entry is dues-eligible; `tier == Size` additionally means it cleared the +/// size bar. `size_vouches`/`dues_vouches` are the qualifying counts (dues is +/// a superset, so `dues_vouches >= size_vouches`). +#[derive(Debug, Clone, Copy)] +pub struct SubjectEligibility { + /// Highest tier the subject cleared. + pub tier: Tier, + /// Judges vouching at the quoted size. + pub size_vouches: usize, + /// Judges vouching for the dues (any size). + pub dues_vouches: usize, +} + +/// Aggregate collected reports into the eligible subset of `subjects`, tagged +/// by the highest tier each cleared (ADR-0005 v5). +/// +/// `reports` is keyed by the REPORTER (the quote responder that shipped it, +/// already signature-verified and nonce-bound). Judges of a subject are the +/// reporters — never the subject itself — that carry a row for it (an +/// "opinion"); reporters with no row ABSTAIN and are not counted. The bar is +/// the same majority-of-opinions bar for both tiers; only the qualifying +/// numerator differs (size-covering vs dues-only). The returned map contains +/// every subject that cleared at least the dues bar; `tier` distinguishes +/// which. +/// +/// The map is keyed by `(PeerId, quoted_size)`, NOT peer alone: eligibility is +/// size-specific, and the same peer may appear at two quoted sizes in one +/// candidate set. Keying on the pair keeps each `(peer, size)` evaluated and +/// filtered against its own quote, so a size-eligible entry for a small quote +/// can never leak Size-tier admission onto a larger quote for that same peer. +#[must_use] +pub fn eligible_subjects( + subjects: &[(PeerId, u32)], + reports: &HashMap, + policy: &EligibilityPolicy, +) -> HashMap<(PeerId, u32), SubjectEligibility> { + let mut out: HashMap<(PeerId, u32), SubjectEligibility> = HashMap::new(); + for (subject, quoted_count) in subjects { + let subject_bytes = *subject.as_bytes(); + // Majority-of-opinions bar (v4). An "opinion" is a reporter (never the + // subject itself) whose report carries a row for the subject at all; + // reporters with no row ABSTAIN. The SAME opinion denominator and bar + // apply to both tiers — a row that vouches for dues but not size is + // still a non-vouching SIZE opinion, so it counts in the size + // denominator. Only the qualifying numerator differs. Excluding a peer + // therefore requires non-vouching rows from over half its opinionated + // reporters — genuine catches (sticky convictions hold the catcher's + // row in the denominator for a dues period) or a collusion the size of + // the neighbourhood-capture boundary. Abstention cannot suppress. + let mut opinions = 0usize; + let mut size_vouches = 0usize; + let mut dues_vouches = 0usize; + for (reporter, report) in reports { + if reporter == subject { + continue; + } + let Some(row) = report + .rows + .iter() + .find(|row| row.subject_peer_id == subject_bytes) + else { + continue; + }; + opinions += 1; + // dues is a superset of size, so check size first and count both. + if row_qualifies(row, *quoted_count, policy) { + size_vouches += 1; + dues_vouches += 1; + } else if row_has_dues(row, policy) { + dues_vouches += 1; + } + } + let bar = policy.quorum_floor.max(opinions / 2 + 1); + let tier = if size_vouches >= bar { + Some(Tier::Size) + } else if dues_vouches >= bar { + Some(Tier::Dues) + } else { + None + }; + // Structured per-subject decision line — the local-testnet runner + // greps these to build the eligibility timeline. + info!( + target: "adr5::eligibility", + subject = %subject, + quoted_count, + vouches = size_vouches, + dues_vouches, + opinions, + bar, + eligible = tier == Some(Tier::Size), + dues_eligible = tier.is_some(), + "eligibility decision" + ); + if let Some(tier) = tier { + out.insert( + (*subject, *quoted_count), + SubjectEligibility { + tier, + size_vouches, + dues_vouches, + }, + ); + } + } + out +} + +/// Which tier `gate_quoter_set` selected from (ADR-0005 v5). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GateTier { + /// Selected the size-eligible subset (full eligibility, `need` met). + Size, + /// Selected the dues-eligible subset (size relaxed, v5 fallback). + Dues, + /// Neither tier filled `need`: today's ungated rules apply. Degraded + /// security — earned eligibility is bypassed. + Ungated, +} + +impl GateTier { + fn label(self) -> &'static str { + match self { + GateTier::Size => "size-eligible", + GateTier::Dues => "dues-eligible (size relaxed)", + GateTier::Ungated => "ungated", + } + } +} + +/// Apply the ADR-0005 gate to a quoter set at collection time (v5 two-tier +/// fallback). +/// +/// Returns the peers to select from. When enforcing, three tiers are tried in +/// order until one fills `need`: +/// 1. **size-eligible** — subjects a majority of judges vouch for AT THE +/// QUOTED SIZE. Whoever wins downstream selection is fully eligible. +/// 2. **dues-eligible (size relaxed)** — if too few are size-eligible, +/// subjects a majority vouch for on DUES (a clean audited week at any +/// size). This is the v5 fallback for fast network-wide growth, where +/// nobody has a full week at the newly grown size. It STILL excludes +/// fresh identities (no dues) and caught cheaters (fenced/convicted), so +/// it drops only the size requirement; ADR-0004 independently forces a +/// current-size proof at payment, so a node can't be paid for a size it +/// doesn't hold. Every size-eligible node is in this tier too. +/// 3. **ungated** — neither tier fills `need`: today's rules apply. This is +/// a degraded-SECURITY mode (earned eligibility bypassed), logged so +/// operators can see how often the network runs ungated. +/// +/// Observe-only (`!policy.enforce`) never changes selection; it logs which +/// tier it WOULD have used so the rollout calibration can distinguish all +/// three outcomes. +/// +/// NOTE (pre-existing v4 limitation, accepted): `size_count >= need` does not +/// guarantee downstream single-node selection succeeds — the witnessed path +/// searches for a close-group whose median issuer has enough witness support +/// and can still return `None`. This tiering filters the candidate set; it +/// does not drive that selector through the tiers. Merkle has no equivalent +/// issue once the selected tier holds ≥ `need` candidates. +#[must_use] +pub fn gate_quoter_set( + items: Vec, + subject_of: impl Fn(&T) -> (PeerId, u32), + reports: &HashMap, + policy: &EligibilityPolicy, + need: usize, + context: &str, +) -> Vec { + let subjects: Vec<(PeerId, u32)> = items.iter().map(&subject_of).collect(); + let eligible = eligible_subjects(&subjects, reports, policy); + let total = items.len(); + // dues set is a superset of the size set; count both. + let size_count = eligible.values().filter(|e| e.tier == Tier::Size).count(); + let dues_count = eligible.len(); + + // The tier that WOULD be selected (also what observe-only reports). + let tier = if size_count >= need { + GateTier::Size + } else if dues_count >= need { + GateTier::Dues + } else { + GateTier::Ungated + }; + + if !policy.enforce { + info!( + "ADR-0005 observe-only [{context}]: would use {} — size-eligible {size_count}/{total}, \ + dues-eligible {dues_count}/{total} (need {need}, floor {}, reports {}); \ + selection unchanged", + tier.label(), + policy.quorum_floor, + reports.len(), + ); + return items; + } + + match tier { + GateTier::Size => { + info!( + "ADR-0005 gate [{context}]: size-eligible — selecting among {size_count} \ + (dropped {} not size-eligible)", + total - size_count, + ); + items + .into_iter() + .filter(|item| { + eligible + .get(&subject_of(item)) + .is_some_and(|e| e.tier == Tier::Size) + }) + .collect() + } + GateTier::Dues => { + // Too few size-eligible; fall back to the dues-eligible subset, + // which INCLUDES the size-eligible ones. No further ranking: + // single-node re-sorts by price/distance and merkle by closeness, + // and preferring the dues tier over ungated IS the "most dues + // done" preference the ADR calls for. + info!( + "ADR-0005 gate [{context}]: DUES-ELIGIBLE (size relaxed) — only \ + {size_count} size-eligible < need {need}; selecting among {dues_count} \ + with a clean audited week at any size (dropped {} without dues)", + total - dues_count, + ); + items + .into_iter() + .filter(|item| eligible.contains_key(&subject_of(item))) + .collect() + } + GateTier::Ungated => { + info!( + "ADR-0005 gate [{context}]: UNGATED (degraded security) — only \ + {size_count} size-eligible / {dues_count} dues-eligible < need {need}; \ + paying under today's rules, earned eligibility bypassed" + ); + debug!( + "ADR-0005 ungated [{context}] eligible set: {:?}", + eligible + .keys() + .map(|(peer, size)| format!("{peer}@{size}")) + .collect::>() + ); + items + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + use super::*; + use ant_protocol::payment::AuditReportDay; + + fn policy() -> EligibilityPolicy { + EligibilityPolicy { + min_distinct_days: 7, + max_recency_days: 1, + size_slack: 2, + window_days: 14, + quorum_floor: 3, + enforce: true, + } + } + + fn clean_row(subject: [u8; 32], days: u16, passes_per_day: u32, size: u32) -> AuditReportRow { + AuditReportRow { + subject_peer_id: subject, + days: (0..days) + .map(|age| AuditReportDay { + age_days: age, + passes: passes_per_day, + max_passed_key_count: size, + }) + .collect(), + fenced: false, + convicted: false, + } + } + + #[test] + fn a_week_of_covering_passes_qualifies() { + let row = clean_row([1; 32], 7, 3, 1000); + assert!(row_qualifies(&row, 1000, &policy())); + assert!( + row_qualifies(&row, 2000, &policy()), + "2x slack covers a doubled quote" + ); + assert!( + !row_qualifies(&row, 2001, &policy()), + "beyond slack the size is not covered" + ); + } + + #[test] + fn too_few_days_or_stale_fails() { + assert!(!row_qualifies( + &clean_row([1; 32], 6, 5, 1000), + 1000, + &policy() + )); + let mut stale = clean_row([1; 32], 7, 5, 1000); + for day in &mut stale.days { + day.age_days += 2; // freshest is now 2 days old + } + assert!(!row_qualifies(&stale, 1000, &policy())); + } + + #[test] + fn fence_and_conviction_disqualify() { + let mut row = clean_row([1; 32], 7, 5, 1000); + row.fenced = true; + assert!(!row_qualifies(&row, 1000, &policy())); + let mut row = clean_row([1; 32], 7, 5, 1000); + row.convicted = true; + assert!(!row_qualifies(&row, 1000, &policy())); + } + + #[test] + fn grind_small_cash_big_fails_day_coverage() { + // A week of passes at size 10, then one fresh pass at size 10_000: + // quoting 10_000 must NOT qualify (only one covering day). + let mut row = clean_row([1; 32], 7, 5, 10); + if let Some(first) = row.days.first_mut() { + first.max_passed_key_count = 10_000; + } + assert!(!row_qualifies(&row, 10_000, &policy())); + // The small history still qualifies small quotes. + assert!(row_qualifies(&row, 10, &policy())); + } + + #[test] + fn duplicate_ages_collapse_to_one_day() { + // 7 entries all claiming age 0: one distinct day, not seven. + let mut row = clean_row([1; 32], 1, 5, 1000); + let day = row.days[0]; + row.days = vec![day; 7]; + assert!(!row_qualifies(&row, 1000, &policy())); + } + + #[test] + fn days_beyond_the_client_window_are_ignored() { + // 7 covering days but 3 of them older than the window: only 4 count. + let mut row = clean_row([1; 32], 7, 5, 1000); + for day in row.days.iter_mut().skip(4) { + day.age_days += 100; + } + assert!(!row_qualifies(&row, 1000, &policy())); + } + + fn report_from(reporter_byte: u8, rows: Vec) -> (PeerId, AuditReport) { + let reporter = PeerId::from_bytes([reporter_byte; 32]); + ( + reporter, + AuditReport { + reporter_peer_id: [reporter_byte; 32], + nonce: [0; 32], + rows, + signature: Vec::new(), + }, + ) + } + + #[test] + fn relative_bar_aggregation_and_self_vouch_exclusion() { + let subject = PeerId::from_bytes([9; 32]); + let subjects = vec![(subject, 1000u32)]; + let good_row = clean_row([9; 32], 7, 5, 1000); + + // Two qualifying observers: below the floor of 3. + let mut reports: HashMap = [ + report_from(1, vec![good_row.clone()]), + report_from(2, vec![good_row.clone()]), + ] + .into_iter() + .collect(); + assert!(eligible_subjects(&subjects, &reports, &policy()).is_empty()); + + // A self-report must not count toward the bar. + let (self_reporter, self_report) = report_from(9, vec![good_row.clone()]); + reports.insert(self_reporter, self_report); + assert!(eligible_subjects(&subjects, &reports, &policy()).is_empty()); + + // A third independent observer clears the floor (3 reporters, bar = + // max(3, 3-2) = 3, vouches 3) at the SIZE tier. + let (r3, rep3) = report_from(3, vec![good_row.clone()]); + reports.insert(r3, rep3); + let eligible = eligible_subjects(&subjects, &reports, &policy()); + let e = eligible.get(&(subject, 1000u32)).expect("size-eligible"); + assert_eq!(e.tier, Tier::Size); + assert_eq!(e.size_vouches, 3); + + // v4 MAJORITY-OF-OPINIONS bar: reporters with NO row for the subject + // abstain — they must not raise the bar (no shortfall to calibrate). + for r in 4..=10u8 { + let (rr, rep_empty) = report_from(r, Vec::new()); + reports.insert(rr, rep_empty); + } + let eligible = eligible_subjects(&subjects, &reports, &policy()); + let e = eligible + .get(&(subject, 1000u32)) + .expect("abstainers don't suppress"); + assert_eq!( + e.tier, + Tier::Size, + "abstaining reporters must not suppress (3 opinions, 3 vouches)" + ); + assert_eq!(e.size_vouches, 3); + + // But non-vouching ROWS count as opinions: catchers with sticky + // convicted rows push the subject below the majority. + let mut convicted_row = clean_row([9; 32], 7, 5, 1000); + convicted_row.convicted = true; + convicted_row.days.clear(); + for r in 4..=8u8 { + let (rr, repv) = report_from(r, vec![convicted_row.clone()]); + reports.insert(rr, repv); + } + // 8 opinions (3 clean + 5 convicted), bar = 8/2+1 = 5 > 3 vouches. + assert!( + eligible_subjects(&subjects, &reports, &policy()).is_empty(), + "a majority of non-vouching rows must exclude the subject" + ); + } + + #[test] + fn gate_filters_only_when_enforcing_with_enough_eligible() { + let subject_a = PeerId::from_bytes([9; 32]); + let subject_b = PeerId::from_bytes([8; 32]); + let items = vec![(subject_a, 1000u32), (subject_b, 1000u32)]; + let good_row_a = clean_row([9; 32], 7, 5, 1000); + let reports: HashMap = [ + report_from(1, vec![good_row_a.clone()]), + report_from(2, vec![good_row_a.clone()]), + report_from(3, vec![good_row_a]), + ] + .into_iter() + .collect(); + + // Enforcing, need 1, subject_a eligible → only subject_a survives. + let kept = gate_quoter_set(items.clone(), |item| *item, &reports, &policy(), 1, "test"); + assert_eq!(kept, vec![(subject_a, 1000u32)]); + + // Need 2 but only 1 eligible → degraded, unchanged. + let kept = gate_quoter_set(items.clone(), |item| *item, &reports, &policy(), 2, "test"); + assert_eq!(kept.len(), 2); + + // Observe-only → unchanged even with an eligible subset. + let mut observe = policy(); + observe.enforce = false; + let kept = gate_quoter_set(items, |item| *item, &reports, &observe, 1, "test"); + assert_eq!(kept.len(), 2); + } + + // ------------------------------------------------------------------ + // ADR-0005 v5: dues-eligibility (size-relaxed tier 2) + // ------------------------------------------------------------------ + + #[test] + fn dues_ignores_size_but_keeps_the_week() { + // A clean week at size 1 (a small node) has done its dues, but is NOT + // size-eligible for a 100k-chunk quote. + let row = clean_row([1; 32], 7, 5, 1); + assert!( + row_has_dues(&row, &policy()), + "a clean week is dues, any size" + ); + assert!( + !row_qualifies(&row, 100_000, &policy()), + "a small node is not size-eligible for a huge quote" + ); + assert!( + row_qualifies(&row, 1, &policy()), + "it IS size-eligible at its own small size" + ); + } + + #[test] + fn dues_still_excludes_fresh_and_caught() { + // Fresh identity: no pass days at all -> neither dues nor size. + let fresh = clean_row([1; 32], 0, 0, 1000); + assert!(!row_has_dues(&fresh, &policy())); + assert!(!row_qualifies(&fresh, 1, &policy())); + + // Convicted: fails both, even with a full week of days underneath. + let mut convicted = clean_row([1; 32], 7, 5, 1000); + convicted.convicted = true; + assert!(!row_has_dues(&convicted, &policy())); + assert!(!row_qualifies(&convicted, 1000, &policy())); + + // Fenced: fails both. + let mut fenced = clean_row([1; 32], 7, 5, 1000); + fenced.fenced = true; + assert!(!row_has_dues(&fenced, &policy())); + assert!(!row_qualifies(&fenced, 1000, &policy())); + } + + #[test] + fn dues_is_a_superset_of_size_including_recency() { + // codex-flagged counterexample class: 7 small passes at ages 0-6 plus + // ONE covering pass at age 13. This has dues (7 distinct recent pass + // days) but is NOT size-eligible for a big quote (only 1 covering day, + // and it is stale) — the split must NOT collapse to + // has_dues && covers_somewhere. + let mut row = clean_row([1; 32], 7, 5, 1); // ages 0..6, size 1 + row.days.push(AuditReportDay { + age_days: 13, + passes: 5, + max_passed_key_count: 100_000, + }); + assert!(row_has_dues(&row, &policy()), "7 recent pass days = dues"); + assert!( + !row_qualifies(&row, 100_000, &policy()), + "one stale covering day is NOT size-eligibility" + ); + } + + #[test] + fn gate_falls_back_to_dues_when_size_short_but_never_admits_bad() { + // 3 judges, all with a clean week at SMALL size 1; a big quote (size + // 100k) makes nobody size-eligible, but all are dues-eligible. + let subject = PeerId::from_bytes([9; 32]); + let small_week = clean_row([9; 32], 7, 5, 1); + let mut reports: HashMap = (1..=3u8) + .map(|r| report_from(r, vec![small_week.clone()])) + .collect(); + + let big = vec![(subject, 100_000u32)]; + let elig = eligible_subjects(&big, &reports, &policy()); + assert_eq!( + elig.get(&(subject, 100_000u32)).map(|e| e.tier), + Some(Tier::Dues), + "size short, dues met -> Dues tier" + ); + // Enforcing gate at big size keeps the dues-eligible node. + let kept = gate_quoter_set(big.clone(), |i| *i, &reports, &policy(), 1, "test"); + assert_eq!(kept, big, "dues fallback keeps the node"); + + // A convicted judge-target is NOT dues-eligible even under fallback: + // every judge carries a convicted row for it. + let convicted_subject = PeerId::from_bytes([7; 32]); + let mut conv = clean_row([7; 32], 7, 5, 1); + conv.convicted = true; + for r in 1..=3u8 { + if let Some(rep) = reports.get_mut(&PeerId::from_bytes([r; 32])) { + rep.rows.push(conv.clone()); + } + } + let elig = eligible_subjects(&[(convicted_subject, 100_000u32)], &reports, &policy()); + assert!( + !elig.contains_key(&(convicted_subject, 100_000u32)), + "a convicted node fails BOTH tiers" + ); + } + + #[test] + fn size_eligible_subjects_are_also_dues_tier_members() { + // A size-eligible node reports tier Size but is counted in the dues + // set too (dues_vouches >= size_vouches). gate_quoter_set's tier-2 + // filter must include it. + let subject = PeerId::from_bytes([9; 32]); + let full = clean_row([9; 32], 7, 5, 1000); + let reports: HashMap = (1..=3u8) + .map(|r| report_from(r, vec![full.clone()])) + .collect(); + let elig = eligible_subjects(&[(subject, 1000u32)], &reports, &policy()); + let e = elig.get(&(subject, 1000u32)).expect("eligible"); + assert_eq!(e.tier, Tier::Size); + assert_eq!(e.size_vouches, 3); + assert_eq!(e.dues_vouches, 3, "size vouches are also dues vouches"); + } + + #[test] + fn same_peer_at_two_sizes_never_leaks_size_tier_across_quotes() { + // The eligibility map is keyed by (peer, size). If it were keyed by peer + // alone, one peer appearing at a small (size-eligible) and a large + // (dues-only) quote would collapse to one entry, and the small quote's + // Size tier could admit the large quote through the size-only filter. + // With a clean week at size 1 only: + // (S, 1) -> Size-eligible + // (S, 100_000) -> Dues-eligible only (1 * slack < 100_000) + let subject = PeerId::from_bytes([9; 32]); + let small_week = clean_row([9; 32], 7, 5, 1); + let reports: HashMap = (1..=3u8) + .map(|r| report_from(r, vec![small_week.clone()])) + .collect(); + + // The large quote is evaluated on its own key: Dues, not Size. + let elig = eligible_subjects( + &[(subject, 1u32), (subject, 100_000u32)], + &reports, + &policy(), + ); + assert_eq!( + elig.get(&(subject, 1u32)).map(|e| e.tier), + Some(Tier::Size), + "the small quote is size-eligible" + ); + assert_eq!( + elig.get(&(subject, 100_000u32)).map(|e| e.tier), + Some(Tier::Dues), + "the large quote is dues-only, regardless of the small quote's tier" + ); + + // The Size-tier gate over BOTH quotes must keep only the small one — the + // large quote has zero size vouches and must not ride the peer's small + // quote into the size tier. + let both = vec![(subject, 1u32), (subject, 100_000u32)]; + let kept = gate_quoter_set(both, |i| *i, &reports, &policy(), 1, "test"); + assert_eq!( + kept, + vec![(subject, 1u32)], + "size gate admits only the size-eligible (peer,size) pair" + ); + } +} diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index af0e44ba..6383a4db 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -18,8 +18,11 @@ use ant_protocol::payment::commitment::{ MAX_COMMITMENT_SIDECAR_BYTES, }; use ant_protocol::payment::{ - calculate_price, serialize_merkle_proof, verify_merkle_candidate_signature, + calculate_price, serialize_merkle_proof, verify_audit_report, + verify_merkle_candidate_signature, AuditReport, MAX_AUDIT_REPORT_BYTES, }; + +use crate::data::client::eligibility::{self, EligibilityPolicy}; use ant_protocol::transport::PeerId; use ant_protocol::{ compute_address, send_and_await_chunk_response, ChunkMessage, ChunkMessageBody, @@ -706,7 +709,18 @@ impl Client { let timeout = Duration::from_secs(self.config().quote_timeout_secs); // Query extra peers to handle validation failures (bad sigs, wrong type, etc.) - let query_count = CANDIDATES_PER_POOL * 2; + // ADR-0005: when the eligibility gate is ENFORCING, widen the query so + // the gate has eligible substitutes beyond the closest 32 — exactly the + // single-node rationale ("a single ineligible fast responder would + // force degraded mode"), and it applies at least as strongly here: + // merkle needs 16 eligible candidates vs single-node's 7, so it is the + // MORE likely path to spurious degraded/ungated selection. Observe-only + // keeps the fixed over-query (and byte cost) unchanged. + let query_count = if EligibilityPolicy::default().enforce { + CANDIDATES_PER_POOL * 4 + } else { + CANDIDATES_PER_POOL * 2 + }; let mut remote_peers = self .network() .find_closest_peers(address, query_count) @@ -736,11 +750,15 @@ impl Client { for (peer_id, peer_addrs) in &remote_peers { let request_id = self.next_request_id(); + // ADR-0005: fresh nonce per candidate request; the responder's + // audit report must echo (and sign) it. + let report_nonce: [u8; 32] = rand::random(); let request = MerkleCandidateQuoteRequest { address: *address, data_type, data_size, merkle_payment_timestamp, + report_nonce, }; let message = ChunkMessage { request_id, @@ -772,12 +790,15 @@ impl Client { MerkleCandidateQuoteResponse::Success { candidate_node, commitment, + audit_report, }, ) => { match rmp_serde::from_slice::( &candidate_node, ) { - Ok(node) => Some(Ok((node, commitment))), + Ok(node) => { + Some(Ok((node, commitment, audit_report, report_nonce))) + } Err(e) => Some(Err(Error::Serialization(format!( "Failed to deserialize candidate node from {peer_id_clone}: {e}" )))), @@ -829,7 +850,15 @@ impl Client { impl std::future::Future< Output = ( PeerId, - std::result::Result<(MerklePaymentCandidateNode, Option>), Error>, + std::result::Result< + ( + MerklePaymentCandidateNode, + Option>, + Option>, + [u8; 32], + ), + Error, + >, ), >, >, @@ -837,11 +866,14 @@ impl Client { merkle_payment_timestamp: u64, ) -> Result<[MerklePaymentCandidateNode; CANDIDATES_PER_POOL]> { let mut valid: Vec<(PeerId, MerklePaymentCandidateNode)> = Vec::new(); + // ADR-0005: verified audit reports by reporter, for the pool-selection + // eligibility gate below. Dropped after selection, never forwarded. + let mut reports: HashMap = HashMap::new(); let mut failures: Vec = Vec::new(); while let Some((peer_id, result)) = futures.next().await { match result { - Ok((candidate, commitment)) => { + Ok((candidate, commitment, audit_report, report_nonce)) => { if !verify_merkle_candidate_signature(&candidate) { warn!("Invalid ML-DSA-65 signature from merkle candidate {peer_id}"); failures.push(format!("{peer_id}: invalid signature")); @@ -880,6 +912,33 @@ impl Client { failures.push(format!("{peer_id}: bad commitment binding ({detail})")); continue; } + // ADR-0005: resolve the candidate's signed audit report. + // Auxiliary testimony about OTHER peers — absent or + // unverifiable never invalidates this candidate; it just + // vouches for nobody this round. + if let Some(bytes) = &audit_report { + if bytes.len() <= MAX_AUDIT_REPORT_BYTES { + match rmp_serde::from_slice::(bytes) { + Ok(report) + if verify_audit_report( + &report, + &candidate.pub_key, + candidate_peer.as_bytes(), + &report_nonce, + ) => + { + reports.insert(candidate_peer, report); + } + _ => warn!( + "Ignoring invalid audit report from merkle candidate {peer_id}" + ), + } + } else { + warn!( + "Ignoring oversized audit report from merkle candidate {peer_id}" + ); + } + } valid.push((candidate_peer, candidate)); } Err(e) => { @@ -900,6 +959,22 @@ impl Client { let target_peer = PeerId::from_bytes(*target_address); valid.sort_by_key(|(peer_id, _)| peer_id.xor_distance(&target_peer)); + // ADR-0005 gate at pool composition: when enforcing AND a full pool of + // eligible candidates exists, the pool is filled eligible-only (order + // stays closest-first, keeping the storer's 13-of-16 close-set check + // satisfiable) — so whichever candidate the contract picks is + // eligible. Otherwise today's closest-16 applies (observe-only or + // explicitly degraded), logged inside the gate. + let valid = eligibility::gate_quoter_set( + valid, + |(peer_id, candidate)| (*peer_id, candidate.committed_key_count), + &reports, + &EligibilityPolicy::default(), + CANDIDATES_PER_POOL, + "merkle-pool", + ); + drop(reports); + let candidates: Vec = valid .into_iter() .take(CANDIDATES_PER_POOL) diff --git a/ant-core/src/data/client/mod.rs b/ant-core/src/data/client/mod.rs index 5214df9b..3cb487b1 100644 --- a/ant-core/src/data/client/mod.rs +++ b/ant-core/src/data/client/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod cached_merkle; pub(crate) mod cached_single; pub mod chunk; pub mod data; +pub mod eligibility; pub mod file; pub mod merkle; pub mod payment; diff --git a/ant-core/src/data/client/quote.rs b/ant-core/src/data/client/quote.rs index 3143d71d..23791dc2 100644 --- a/ant-core/src/data/client/quote.rs +++ b/ant-core/src/data/client/quote.rs @@ -3,6 +3,7 @@ //! Handles requesting storage quotes from network nodes and //! managing payment for data storage. +use crate::data::client::eligibility::{self, EligibilityPolicy}; use crate::data::client::peer_xor_distance; use crate::data::client::Client; use crate::data::client::PUT_TARGET_WIDTH; @@ -13,6 +14,7 @@ use ant_protocol::payment::commitment::{ commitment_hash, verify_commitment_signature, StorageCommitment, MAX_COMMITMENT_KEY_COUNT, MAX_COMMITMENT_SIDECAR_BYTES, }; +use ant_protocol::payment::{verify_audit_report, AuditReport, MAX_AUDIT_REPORT_BYTES}; use ant_protocol::payment::{verify_quote_content, verify_quote_signature}; use ant_protocol::transport::{ DHTNode, MultiAddr, P2PNode, PeerId, ResponderView, WitnessedCloseGroup, @@ -72,6 +74,12 @@ type QuotedPeer = ( Option>, ); +/// A classified quote response: the verified quote, its price, and the +/// ADR-0004 commitment sidecar. The ADR-0005 audit report travels separately +/// (a sink filled for every VERIFIED responder, including already-stored +/// voters — their testimony counts even though they issue no payable quote). +type ClassifiedQuote = (PaymentQuote, Amount, Option>); + /// Check that a quote's `pub_key` is well-formed and BLAKE3-hashes to the /// claimed `peer_id`. /// @@ -229,13 +237,17 @@ fn quote_commitment_binding_is_valid( /// On success the returned commitment is the opaque signed-commitment blob the /// node shipped with the quote (`None` for a baseline quote), to be forwarded /// as a sidecar in the PUT bundle. +#[allow(clippy::too_many_arguments)] fn classify_quote_response( peer_id: &PeerId, expected_content: &[u8; 32], quote_bytes: &[u8], already_stored: bool, commitment: Option>, -) -> std::result::Result<(PaymentQuote, Amount, Option>), Error> { + audit_report: Option>, + report_nonce: &[u8; 32], + report_sink: &std::sync::Mutex>, +) -> std::result::Result { let payment_quote = rmp_serde::from_slice::(quote_bytes).map_err(|e| { Error::Serialization(format!("Failed to deserialize quote from {peer_id}: {e}")) })?; @@ -292,10 +304,58 @@ fn classify_quote_response( }); } + // ADR-0005: resolve the responder's signed audit report BEFORE the + // already-stored short-circuit — an already-stored voter issues no + // payable quote, but its (fully verified) testimony about other peers + // still counts toward the eligibility quorum. The report is auxiliary: + // absent or unverifiable never invalidates this peer's own quote; it + // just means this responder vouches for nobody this round. The signature + // is checked against the quote's own (already peer-bound) public key, + // and the echoed nonce proves the report was generated for this request. + let report = audit_report.and_then(|bytes| { + // Structured size line — the local-testnet runner greps these for the + // overhead budget check (build/verify timing is measured by the runner). + debug!( + target: "adr5::report_size", + peer = %peer_id, + bytes = bytes.len(), + "audit report received" + ); + if bytes.len() > MAX_AUDIT_REPORT_BYTES { + warn!( + "Ignoring oversized audit report from {peer_id} ({} bytes)", + bytes.len() + ); + return None; + } + let parsed: AuditReport = match rmp_serde::from_slice(&bytes) { + Ok(report) => report, + Err(e) => { + warn!("Ignoring unparseable audit report from {peer_id}: {e}"); + return None; + } + }; + if verify_audit_report( + &parsed, + &payment_quote.pub_key, + peer_id.as_bytes(), + report_nonce, + ) { + Some(parsed) + } else { + warn!("Ignoring audit report from {peer_id} — signature/nonce/shape check failed"); + None + } + }); + if let Ok(mut sink) = report_sink.lock() { + *sink = report; + } + if already_stored { debug!("Peer {peer_id} already has chunk"); return Err(Error::AlreadyStored); } + let price = payment_quote.price; debug!("Received quote from {peer_id}: price = {price}"); Ok((payment_quote, price, commitment)) @@ -331,10 +391,15 @@ async fn request_store_quote_from_peer( data_type: u32, per_peer_timeout: Duration, ) -> StoreQuoteRequestResult { + // ADR-0005: a fresh random nonce per request; the responder's audit + // report must echo (and sign) it, so reports cannot be precomputed or + // replayed across requests. + let report_nonce: [u8; 32] = rand::random(); let request = ChunkQuoteRequest { address, data_size, data_type, + report_nonce, }; let message = ChunkMessage { request_id, @@ -350,10 +415,15 @@ async fn request_store_quote_from_peer( Err(Error::Protocol(format!( "Failed to encode quote request for {peer_id}: {e}" ))), + None, ); } }; + // Sink for the responder's verified audit report: filled by the + // classifier for every VERIFIED responder, including already-stored + // voters whose classification is an Err(AlreadyStored). + let report_sink = std::sync::Mutex::new(None); let result = send_and_await_chunk_response( &node, &peer_id, @@ -366,12 +436,16 @@ async fn request_store_quote_from_peer( quote, already_stored, commitment, + audit_report, }) => Some(classify_quote_response( &peer_id, &address, "e, already_stored, commitment, + audit_report, + &report_nonce, + &report_sink, )), ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error(e)) => Some(Err( Error::Protocol(format!("Quote error from {peer_id}: {e}")), @@ -383,20 +457,29 @@ async fn request_store_quote_from_peer( ) .await; - (peer_id, peer_addrs, result) + let report = report_sink.into_inner().unwrap_or_default(); + (peer_id, peer_addrs, result, report) } #[allow(clippy::too_many_arguments)] fn record_store_quote_result( peer_id: PeerId, addrs: Vec, - quote_result: Result<(PaymentQuote, Amount, Option>)>, + quote_result: Result, + report: Option, address: &[u8; 32], quotes: &mut Vec, + reports: &mut HashMap, already_stored_peers: &mut Vec<(PeerId, [u8; 32])>, failures: &mut Vec, bad_quote_count: &mut usize, ) { + // ADR-0005: keep every VERIFIED responder's report for the + // collection-time eligibility decision (already-stored voters testify + // too); dropped after selection, never forwarded. + if let Some(report) = report { + reports.insert(peer_id, report); + } match quote_result { Ok((quote, price, commitment)) => { quotes.push((peer_id, addrs, quote, price, commitment)); @@ -417,11 +500,12 @@ fn record_store_quote_result( } fn witnessed_quote_launch_budget( + target: usize, successful_quotes: usize, in_flight: usize, remaining_peers: usize, ) -> usize { - CLOSE_GROUP_SIZE + target .saturating_sub(successful_quotes.saturating_add(in_flight)) .min(remaining_peers) } @@ -502,7 +586,8 @@ pub(crate) type StoreQuote = ( type StoreQuoteRequestResult = ( PeerId, Vec, - Result<(PaymentQuote, Amount, Option>)>, + Result, + Option, ); type VotersByPeer = HashMap>; type WitnessedVoteData = (HashMap, VotersByPeer, Vec<(PeerId, usize)>); @@ -1125,6 +1210,9 @@ impl Client { // the closest witnessed peers first and only falls back to further // witnessed peers when a closer peer fails to produce a usable quote. let mut quotes = Vec::with_capacity(peer_query_count); + // ADR-0005: verified audit reports by reporter, for the collection-time + // eligibility gate below. Dropped after selection. + let mut reports: HashMap = HashMap::new(); let mut already_stored_peers: Vec<(PeerId, [u8; 32])> = Vec::new(); let mut failures: Vec = Vec::new(); @@ -1139,6 +1227,19 @@ impl Client { QuoteSelectionPolicy::WitnessedMedianVoters { .. } ); + // ADR-0005: when the eligibility gate is ENFORCING, collect quotes + // from the whole witnessed candidate set instead of stopping at the + // first CLOSE_GROUP_SIZE successes — the gate needs eligible + // substitutes beyond the closest 7, or a single ineligible fast + // responder would force degraded mode. Observe-only keeps today's + // early-stop behaviour (and byte cost) unchanged. + let eligibility_policy = EligibilityPolicy::default(); + let collection_target = if eligibility_policy.enforce { + remote_peers.len().max(CLOSE_GROUP_SIZE) + } else { + CLOSE_GROUP_SIZE + }; + if staged_witnessed_collection { let mut quote_futures = FuturesUnordered::new(); let mut next_peer_index = 0usize; @@ -1146,6 +1247,7 @@ impl Client { tokio::time::timeout(overall_timeout, async { loop { let launch_count = witnessed_quote_launch_budget( + collection_target, quotes.len(), quote_futures.len(), remote_peers.len().saturating_sub(next_peer_index), @@ -1165,11 +1267,12 @@ impl Client { )); } - if quotes.len() >= CLOSE_GROUP_SIZE || quote_futures.is_empty() { + if quotes.len() >= collection_target || quote_futures.is_empty() { break; } - let Some((peer_id, addrs, quote_result)) = quote_futures.next().await + let Some((peer_id, addrs, quote_result, report)) = + quote_futures.next().await else { break; }; @@ -1177,8 +1280,10 @@ impl Client { peer_id, addrs, quote_result, + report, address, &mut quotes, + &mut reports, &mut already_stored_peers, &mut failures, &mut bad_quote_count, @@ -1219,13 +1324,17 @@ impl Client { let collect_result: std::result::Result, _> = tokio::time::timeout(overall_timeout, async { - while let Some((peer_id, addrs, quote_result)) = quote_futures.next().await { + while let Some((peer_id, addrs, quote_result, report)) = + quote_futures.next().await + { record_store_quote_result( peer_id, addrs, quote_result, + report, address, &mut quotes, + &mut reports, &mut already_stored_peers, &mut failures, &mut bad_quote_count, @@ -1297,6 +1406,20 @@ impl Client { let quote_count = quotes.len(); let total_responses = quote_count + failure_count + already_stored_count; + // ADR-0005 gate at collection: when enforcing AND a full close group of + // eligible quoters exists, selection runs on the eligible subset — so + // whoever wins the median is eligible. Otherwise (observe-only, or too + // few eligible = explicitly degraded) today's rules apply, logged. + let quotes = eligibility::gate_quoter_set( + quotes, + |(peer_id, _, quote, _, _): &StoreQuote| (*peer_id, quote.committed_key_count), + &reports, + &eligibility_policy, + CLOSE_GROUP_SIZE, + "single-node", + ); + drop(reports); + if quotes.len() >= CLOSE_GROUP_SIZE { let selected_quotes = match quote_selection_policy { QuoteSelectionPolicy::ClosestByDistance => select_closest_quotes(quotes, address), @@ -1358,6 +1481,45 @@ mod tests { use std::time::SystemTime; use xor_name::XorName; + /// P3 guard (codex impl-gate): a VERIFIED report arriving with an + /// `Err(AlreadyStored)` classification must still reach the eligibility + /// report map — already-stored voters testify too. + #[test] + fn already_stored_responder_report_reaches_the_map() { + use ant_protocol::payment::AuditReport; + + let peer = PeerId::from_bytes([3u8; 32]); + let report = AuditReport { + reporter_peer_id: [3u8; 32], + nonce: [1u8; 32], + rows: Vec::new(), + signature: Vec::new(), + }; + let mut quotes = Vec::new(); + let mut reports = HashMap::new(); + let mut already = Vec::new(); + let mut failures = Vec::new(); + let mut bad = 0usize; + record_store_quote_result( + peer, + Vec::new(), + Err(Error::AlreadyStored), + Some(report), + &[9u8; 32], + &mut quotes, + &mut reports, + &mut already, + &mut failures, + &mut bad, + ); + assert!(quotes.is_empty()); + assert_eq!(already.len(), 1, "already-stored vote recorded"); + assert!( + reports.contains_key(&peer), + "already-stored responder's testimony must reach the report map" + ); + } + /// A real ML-DSA-65 keypair plus its derived peer ID. struct Keypair { peer_id: PeerId, @@ -1623,30 +1785,51 @@ mod tests { #[test] fn witnessed_quote_launch_budget_keeps_exact_quote_window() { assert_eq!( - witnessed_quote_launch_budget(0, 0, CLOSE_GROUP_SIZE * 2), + witnessed_quote_launch_budget(CLOSE_GROUP_SIZE, 0, 0, CLOSE_GROUP_SIZE * 2), CLOSE_GROUP_SIZE, "initial SNP quote fetch should launch the closest seven peers" ); assert_eq!( - witnessed_quote_launch_budget(1, CLOSE_GROUP_SIZE - 1, CLOSE_GROUP_SIZE), + witnessed_quote_launch_budget( + CLOSE_GROUP_SIZE, + 1, + CLOSE_GROUP_SIZE - 1, + CLOSE_GROUP_SIZE + ), 0, "a successful quote should not launch an extra fallback" ); assert_eq!( - witnessed_quote_launch_budget(0, CLOSE_GROUP_SIZE - 1, CLOSE_GROUP_SIZE), + witnessed_quote_launch_budget( + CLOSE_GROUP_SIZE, + 0, + CLOSE_GROUP_SIZE - 1, + CLOSE_GROUP_SIZE + ), 1, "a failed in-flight quote should launch the next closest fallback" ); assert_eq!( - witnessed_quote_launch_budget(CLOSE_GROUP_SIZE - 1, 0, 3), + witnessed_quote_launch_budget(CLOSE_GROUP_SIZE, CLOSE_GROUP_SIZE - 1, 0, 3), 1, "only one more peer is needed for the seventh quote" ); assert_eq!( - witnessed_quote_launch_budget(0, 0, CLOSE_GROUP_SIZE - 1), + witnessed_quote_launch_budget(CLOSE_GROUP_SIZE, 0, 0, CLOSE_GROUP_SIZE - 1), CLOSE_GROUP_SIZE - 1, "launch budget is capped by remaining candidates" ); + // ADR-0005 enforcement widens the target to the whole candidate set. + assert_eq!( + witnessed_quote_launch_budget( + CLOSE_GROUP_SIZE * 2, + CLOSE_GROUP_SIZE, + 0, + CLOSE_GROUP_SIZE + ), + CLOSE_GROUP_SIZE, + "an enforcing client keeps collecting past the first seven" + ); } #[test] @@ -2264,7 +2447,16 @@ mod tests { let content = [7u8; 32]; let (peer_id, quote) = signed_baseline_quote(content); let bytes = serialize_quote("e); - let result = classify_quote_response(&peer_id, &content, &bytes, false, None); + let result = classify_quote_response( + &peer_id, + &content, + &bytes, + false, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ); match result { Ok((q, price, commitment)) => { assert_eq!(q.pub_key, quote.pub_key); @@ -2283,7 +2475,16 @@ mod tests { let (peer_id, mut quote) = signed_baseline_quote(content); quote.signature = vec![0u8; quote.signature.len()]; // corrupt the signature let bytes = serialize_quote("e); - let result = classify_quote_response(&peer_id, &content, &bytes, false, None); + let result = classify_quote_response( + &peer_id, + &content, + &bytes, + false, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ); assert!( matches!(result, Err(Error::BadQuoteBinding { .. })), "a quote with an invalid signature must be rejected; got {result:?}" @@ -2295,7 +2496,16 @@ mod tests { // A validly-signed quote for a DIFFERENT address is dropped before pay. let (peer_id, quote) = signed_baseline_quote([7u8; 32]); let bytes = serialize_quote("e); - let result = classify_quote_response(&peer_id, &[9u8; 32], &bytes, false, None); + let result = classify_quote_response( + &peer_id, + &[9u8; 32], + &bytes, + false, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ); assert!( matches!(result, Err(Error::BadQuoteBinding { .. })), "a quote for the wrong content must be rejected; got {result:?}" @@ -2306,7 +2516,16 @@ mod tests { fn classifier_rejects_crossed_keypair_with_typed_error() { let (peer_id, _, quote, _, _) = bad_quote_real(); let bytes = serialize_quote("e); - let result = classify_quote_response(&peer_id, &[0u8; 32], &bytes, false, None); + let result = classify_quote_response( + &peer_id, + &[0u8; 32], + &bytes, + false, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ); match result { Err(Error::BadQuoteBinding { peer_id: pid, @@ -2337,7 +2556,16 @@ mod tests { let (peer_id, _, quote, _, _) = bad_quote_real(); let bytes = serialize_quote("e); // The peer claims already_stored=true, but its quote has a crossed key. - let result = classify_quote_response(&peer_id, &[0u8; 32], &bytes, true, None); + let result = classify_quote_response( + &peer_id, + &[0u8; 32], + &bytes, + true, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ); assert!( matches!(result, Err(Error::BadQuoteBinding { .. })), "crossed-key peer must be classified BadQuoteBinding even when \ @@ -2352,7 +2580,16 @@ mod tests { let content = [7u8; 32]; let (peer_id, quote) = signed_baseline_quote(content); let bytes = serialize_quote("e); - let result = classify_quote_response(&peer_id, &content, &bytes, true, None); + let result = classify_quote_response( + &peer_id, + &content, + &bytes, + true, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ); assert!( matches!(result, Err(Error::AlreadyStored)), "honest peer's already_stored vote must be honoured; got {result:?}" @@ -2363,7 +2600,16 @@ mod tests { fn classifier_returns_serialization_error_on_bad_bytes() { let (peer_id, _, _, _, _) = good_quote_real(); let garbage = b"this is not a valid msgpack PaymentQuote".to_vec(); - let result = classify_quote_response(&peer_id, &[0u8; 32], &garbage, false, None); + let result = classify_quote_response( + &peer_id, + &[0u8; 32], + &garbage, + false, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ); assert!( matches!(result, Err(Error::Serialization(_))), "garbage bytes must produce a Serialization error; got {result:?}" @@ -2385,8 +2631,17 @@ mod tests { for (peer_id, quote) in &responders { let bytes = serialize_quote(quote); let storer_verdict = storer_binding_would_accept(peer_id, quote); - let classifier_verdict = - classify_quote_response(peer_id, &content, &bytes, false, None).is_ok(); + let classifier_verdict = classify_quote_response( + peer_id, + &content, + &bytes, + false, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ) + .is_ok(); assert_eq!( classifier_verdict, storer_verdict, "classifier and storer-binding-spec must agree on every responder \ @@ -2642,7 +2897,16 @@ mod tests { .as_bytes() .to_vec(); let bytes = serialize_quote("e); - let result = classify_quote_response(&kp.peer_id, &content, &bytes, false, None); + let result = classify_quote_response( + &kp.peer_id, + &content, + &bytes, + false, + None, + None, + &[0u8; 32], + &std::sync::Mutex::new(None), + ); assert!( matches!(result, Err(Error::BadQuoteCommitment { .. })), "off-curve quote must be dropped as BadQuoteCommitment; got {result:?}" diff --git a/ant-core/tests/e2e_adr0005.rs b/ant-core/tests/e2e_adr0005.rs new file mode 100644 index 00000000..345de123 --- /dev/null +++ b/ant-core/tests/e2e_adr0005.rs @@ -0,0 +1,529 @@ +//! ADR-0005 E2E: earned reward eligibility over real QUIC + Anvil. +//! +//! MiniTestnet nodes run no replication engine, so no real audits fire; each +//! node instead carries a real `AuditTally` (wired as its quote generator's +//! report source) that the tests SEED with controlled facts. That makes the +//! full wire path — tally → signed report on the quote response → client +//! verification → eligibility gate at collection — deterministic: what a +//! quorum of observers testifies is exactly what the test wrote. +//! +//! Timing note: histories are seeded with real past timestamps (1 day = +//! 86 400 s, the production bucket), so no time compression is involved here. +//! The emergent audit-driven timing ("honest nodes qualify in ≈ a week") is +//! Layer 3's job — the process-level local testnet with compressed days. + +mod support; + +use ant_core::data::client::merkle::PaymentMode; +use ant_core::data::{compute_address, Client}; +use bytes::Bytes; +use serial_test::serial; +use std::sync::Arc; +use support::{test_client_config, MiniTestnet, DEFAULT_NODE_COUNT}; + +const COMMITTED_KEYS: u32 = 700; +const WEEK_DAYS: u64 = 7; + +/// Sets the ADR-0005 enforcement env for one test and clears it on drop +/// (tests are `#[serial]`, so no cross-test interleaving). +struct EnforceGuard; +impl EnforceGuard { + fn set() -> Self { + std::env::set_var("ADR5_ENFORCE", "1"); + Self + } +} +impl Drop for EnforceGuard { + fn drop(&mut self) { + std::env::remove_var("ADR5_ENFORCE"); + } +} + +/// Seed a full clean week for every (observer, subject) pair except subjects +/// listed in `skip` — those get NO testimony anywhere (fresh/unproven nodes). +fn seed_all_except(testnet: &MiniTestnet, skip: &[usize]) { + for subject in 0..testnet.nodes.len() { + if skip.contains(&subject) { + continue; + } + testnet.seed_history_for_all_observers(subject, COMMITTED_KEYS, WEEK_DAYS); + } +} + +/// XOR distance between a peer id and an address, comparable byte-wise. +fn xor_distance(peer: &[u8; 32], addr: &[u8; 32]) -> [u8; 32] { + let mut d = [0u8; 32]; + for i in 0..32 { + d[i] = peer[i] ^ addr[i]; + } + d +} + +/// Mine chunk content whose address puts `target` inside the closest +/// `within` peers of the testnet (excluding `client_index`, which never +/// quotes to itself). With ~14 nodes this converges in a handful of tries. +fn mine_content_with_target_close( + testnet: &MiniTestnet, + target_index: usize, + client_index: usize, + within: usize, +) -> Bytes { + let peer_ids: Vec<(usize, [u8; 32])> = (0..testnet.nodes.len()) + .filter(|i| *i != client_index) + .map(|i| (i, testnet.peer_id_bytes(i))) + .collect(); + for salt in 0u64..100_000 { + let content = Bytes::from(format!("adr-0005 mined payload {salt}")); + let address = compute_address(&content); + let mut by_distance = peer_ids.clone(); + by_distance.sort_by_key(|(_, peer)| xor_distance(peer, &address)); + if by_distance + .iter() + .take(within) + .any(|(i, _)| *i == target_index) + { + return content; + } + } + panic!("could not mine an address near the target peer"); +} + +/// Scenario 1 + 5 (gate-on liveness both ways): with EVERY quoter eligible, +/// enforcement changes nothing observable — a full put/get round-trip works +/// and a close group of quotes is collected. With NOBODY eligible (empty +/// tallies), enforcement degrades to today's rules and the round-trip STILL +/// works: the gate never stalls the network. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr5_enforced_gate_keeps_liveness_all_eligible_and_none_eligible() { + let testnet = + MiniTestnet::start_with_commitments_and_tallies(DEFAULT_NODE_COUNT, COMMITTED_KEYS).await; + let node = testnet.node(3).expect("node 3 exists"); + let client = Client::from_node(Arc::clone(&node), test_client_config()) + .with_wallet(testnet.wallet().clone()); + let _guard = EnforceGuard::set(); + + // Phase A: nobody has testimony yet (all tallies empty) -> degraded mode, + // uploads must still succeed. + let content = Bytes::from("adr-0005 degraded-mode payload"); + let address = compute_address(&content); + let stored = client + .chunk_put(content.clone()) + .await + .expect("degraded-mode put must succeed (the gate never stalls)"); + assert_eq!(stored, address); + + // Phase B: seed a full week for everyone -> gate-on path, everyone + // eligible, round-trip still green. + seed_all_except(&testnet, &[]); + let content = Bytes::from("adr-0005 all-eligible payload"); + let address = compute_address(&content); + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("quote collection under enforcement"); + assert!( + quotes.len() >= ant_protocol::CLOSE_GROUP_SIZE, + "full close group under enforcement, got {}", + quotes.len() + ); + let stored = client + .chunk_put(content.clone()) + .await + .expect("all-eligible enforced put must succeed"); + assert_eq!(stored, address); + let retrieved = client.chunk_get(&address).await.expect("get back"); + assert!(retrieved.is_some(), "stored chunk must be retrievable"); + + testnet.teardown().await; +} + +/// Scenario 3b (forced single-node substitution): one node has NO testimony +/// anywhere and sits inside the closest-7 of a mined address. Enforcement +/// must widen collection, drop it from the payable set, and still assemble a +/// full close group of eligible quotes. Observe-only (default) must keep it. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr5_unproven_quoter_is_substituted_under_enforcement() { + let testnet = + MiniTestnet::start_with_commitments_and_tallies(DEFAULT_NODE_COUNT, COMMITTED_KEYS).await; + let client_index = 0usize; + let fresh_index = 5usize; + seed_all_except(&testnet, &[fresh_index]); + + let node = testnet.node(client_index).expect("client node"); + let client = Client::from_node(Arc::clone(&node), test_client_config()) + .with_wallet(testnet.wallet().clone()); + let content = mine_content_with_target_close( + &testnet, + fresh_index, + client_index, + ant_protocol::CLOSE_GROUP_SIZE, + ); + let address = compute_address(&content); + let fresh_peer = testnet.peer_id_bytes(fresh_index); + + // Observe-only first: the unproven quoter is closest-7, so it appears. + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("observe-only quote collection"); + assert!( + quotes + .iter() + .any(|(peer, _, _, _, _)| *peer.as_bytes() == fresh_peer), + "observe-only keeps the unproven closest-7 quoter" + ); + + // Enforcement: substituted away, full close group still assembled. + let _guard = EnforceGuard::set(); + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("enforced quote collection must not degrade with substitutes available"); + assert_eq!( + quotes.len(), + ant_protocol::CLOSE_GROUP_SIZE, + "a full close group of eligible quoters" + ); + assert!( + quotes + .iter() + .all(|(peer, _, _, _, _)| *peer.as_bytes() != fresh_peer), + "the unproven quoter must be substituted out of the payable set" + ); + + // And the gated payment path completes end to end. + let stored = client + .chunk_put(content.clone()) + .await + .expect("enforced put with substitution must succeed"); + assert_eq!(stored, address); + + testnet.teardown().await; +} + +/// Scenario 2 (conviction resets the dues): a convicted node is excluded even +/// though it had a full week of history; once it re-earns a fresh week of +/// passes (which also clears the outstanding-conviction marker), it is +/// payable again. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr5_convicted_node_excluded_until_it_reearns() { + let testnet = + MiniTestnet::start_with_commitments_and_tallies(DEFAULT_NODE_COUNT, COMMITTED_KEYS).await; + let client_index = 0usize; + let subject_index = 6usize; + seed_all_except(&testnet, &[]); + + let node = testnet.node(client_index).expect("client node"); + let client = Client::from_node(Arc::clone(&node), test_client_config()) + .with_wallet(testnet.wallet().clone()); + let content = mine_content_with_target_close( + &testnet, + subject_index, + client_index, + ant_protocol::CLOSE_GROUP_SIZE, + ); + let address = compute_address(&content); + let subject_peer = testnet.peer_id_bytes(subject_index); + let _guard = EnforceGuard::set(); + + // Convicted at every observer: the week of history is gone. + testnet.convict_at_all_observers(subject_index, 0); + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("enforced collection with a convicted closest-7 quoter"); + assert!( + quotes + .iter() + .all(|(peer, _, _, _, _)| *peer.as_bytes() != subject_peer), + "a convicted node must not be payable" + ); + + // v4 sticky convictions: a fresh week of passes alone does NOT restore + // eligibility while the marker is outstanding. + testnet.seed_history_for_all_observers(subject_index, COMMITTED_KEYS, WEEK_DAYS); + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("collection inside the sticky period"); + assert!( + quotes + .iter() + .all(|(peer, _, _, _, _)| *peer.as_bytes() != subject_peer), + "the sticky conviction must hold through fresh passes for a dues period" + ); + + // Model "one dues period has since elapsed": re-record the conviction 8 + // days in the past (aging it beyond CONVICTION_STICKY_DAYS zeroes the + // rows again), then re-earn a fresh week on top -> payable again. + testnet.convict_at_all_observers(subject_index, 8); + testnet.seed_history_for_all_observers(subject_index, COMMITTED_KEYS, WEEK_DAYS); + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("collection after the sticky period + re-earned dues"); + assert!( + quotes + .iter() + .any(|(peer, _, _, _, _)| *peer.as_bytes() == subject_peer), + "a re-earned node must be payable again (conviction is one dues period, not a ban)" + ); + + testnet.teardown().await; +} + +/// Silent-hopper fencing: a fenced node (unanswered challenge on a monetized +/// pin, no pass since) is excluded exactly like a convicted one, but keeps +/// its history — a single fresh pass restores it. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr5_fenced_node_excluded_until_next_pass() { + let testnet = + MiniTestnet::start_with_commitments_and_tallies(DEFAULT_NODE_COUNT, COMMITTED_KEYS).await; + let client_index = 0usize; + let subject_index = 4usize; + seed_all_except(&testnet, &[]); + + let node = testnet.node(client_index).expect("client node"); + let client = Client::from_node(Arc::clone(&node), test_client_config()) + .with_wallet(testnet.wallet().clone()); + let content = mine_content_with_target_close( + &testnet, + subject_index, + client_index, + ant_protocol::CLOSE_GROUP_SIZE, + ); + let address = compute_address(&content); + let subject_peer = testnet.peer_id_bytes(subject_index); + let _guard = EnforceGuard::set(); + + testnet.fence_at_all_observers(subject_index); + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("enforced collection with a fenced closest-7 quoter"); + assert!( + quotes + .iter() + .all(|(peer, _, _, _, _)| *peer.as_bytes() != subject_peer), + "a fenced node must not be payable while the challenge is outstanding" + ); + + // One fresh pass per observer clears the fence; history was kept, so the + // node is immediately payable again (no re-grind — that is conviction's + // job). + testnet.seed_history_for_all_observers(subject_index, COMMITTED_KEYS, 1); + let quotes = client + .get_store_quotes(&address, content.len() as u64, 0) + .await + .expect("collection after the fence clears"); + assert!( + quotes + .iter() + .any(|(peer, _, _, _, _)| *peer.as_bytes() == subject_peer), + "a fence clears on the next pass, restoring the kept history" + ); + + testnet.teardown().await; +} + +/// A tracing writer that appends into a shared buffer, so a test can assert +/// on the client's structured `adr5::*` gate decisions. +#[derive(Clone)] +struct CaptureWriter(std::sync::Arc>>); +impl std::io::Write for CaptureWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().expect("capture lock").extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CaptureWriter { + type Writer = Self; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } +} + +/// Scenario 3c (merkle mixed-eligibility pool): with one unproven node in the +/// candidate field and 16+ eligible candidates available, an enforced merkle +/// upload completes end-to-end — the pool is composed eligible-first and the +/// storers' closeness/payment verification still passes. Non-vacuous by +/// construction: the client's own gate decisions must show the unproven node +/// was EVALUATED in a merkle candidate field and NEVER judged eligible, and +/// at least one merkle-pool gate dropped an ineligible candidate. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr5_merkle_mixed_pool_uploads_under_enforcement() { + // Merkle pools need 16 candidates; 35 nodes matches the merkle e2e suite. + let testnet = MiniTestnet::start_with_commitments_and_tallies(35, COMMITTED_KEYS).await; + let fresh_index = 7usize; + seed_all_except(&testnet, &[fresh_index]); + let fresh_peer_hex = hex::encode(testnet.peer_id_bytes(fresh_index)); + + let node = testnet.node(0).expect("client node"); + let client = Client::from_node(Arc::clone(&node), test_client_config()) + .with_wallet(testnet.wallet().clone()); + let _guard = EnforceGuard::set(); + + // Capture the gate's structured decisions for the duration of the upload. + let capture = CaptureWriter(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))); + let subscriber = tracing_subscriber::fmt() + .with_writer(capture.clone()) + .with_ansi(false) + .with_max_level(tracing::Level::INFO) + .finish(); + let _sub_guard = tracing::subscriber::set_default(subscriber); + + // Upload enough data for several pools so the unproven node lands in at + // least one 32-wide candidate query with overwhelming probability. + let data = Bytes::from(vec![0x5Au8; 2 * 1024 * 1024]); + let result = client + .data_upload_with_mode(data.clone(), PaymentMode::Merkle) + .await + .expect("enforced merkle upload with a mixed-eligibility field must succeed"); + let retrieved = client + .data_download(&result.data_map) + .await + .expect("download back"); + assert_eq!(retrieved, data); + + let captured = String::from_utf8_lossy(&capture.0.lock().expect("capture lock")).to_string(); + let fresh_decisions: Vec<&str> = captured + .lines() + .filter(|l| l.contains("eligibility decision") && l.contains(fresh_peer_hex.as_str())) + .collect(); + assert!( + !fresh_decisions.is_empty(), + "the unproven node must have been evaluated in at least one candidate field" + ); + assert!( + fresh_decisions.iter().all(|l| l.contains("eligible=false")), + "the unproven node must never be judged eligible" + ); + assert!( + captured + .lines() + .any(|l| l.contains("ADR-0005 gate [merkle-pool]") + && !l.contains("dropped 0 ineligible")), + "at least one merkle pool must have dropped an ineligible candidate" + ); + + testnet.teardown().await; +} + +/// v5 fast-growth (tier-2 dues fallback): the whole network grew fast, so +/// nobody has a week of history AT THE CURRENT (grown) size — size-eligibility +/// is empty network-wide. The two-tier gate must then fall back to +/// DUES-eligibility (a clean audited week at any size) and keep paying, WHILE +/// still excluding a fresh no-history node and a convicted node. This is the +/// scenario that motivated the whole two-tier change. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn adr5_fast_growth_falls_back_to_dues_but_excludes_fresh_and_convicted() { + // Nodes commit to (and quote at) the LARGE grown size... + const GROWN_SIZE: u32 = COMMITTED_KEYS; // 700, the current commitment + const SMALL_HISTORY: u32 = 1; // ...but only ever passed audits when small + let testnet = + MiniTestnet::start_with_commitments_and_tallies(DEFAULT_NODE_COUNT, GROWN_SIZE).await; + + let client_index = 0usize; + let fresh_index = 5usize; + let convicted_index = 6usize; + + // Everyone except the fresh node earns a clean week — but at the SMALL + // size, so with slack 2x nobody covers the grown quote (1*2 < 700). All + // are dues-eligible; none are size-eligible. + for subject in 0..testnet.nodes.len() { + if subject == fresh_index { + continue; + } + testnet.seed_history_for_all_observers(subject, SMALL_HISTORY, WEEK_DAYS); + } + // The convicted node has its clean week wiped at every observer. + testnet.convict_at_all_observers(convicted_index, 0); + + let fresh_peer_hex = hex::encode(testnet.peer_id_bytes(fresh_index)); + let convicted_peer_hex = hex::encode(testnet.peer_id_bytes(convicted_index)); + + let node = testnet.node(client_index).expect("client node"); + let client = Client::from_node(Arc::clone(&node), test_client_config()) + .with_wallet(testnet.wallet().clone()); + let _guard = EnforceGuard::set(); + + let capture = CaptureWriter(std::sync::Arc::new(std::sync::Mutex::new(Vec::new()))); + let subscriber = tracing_subscriber::fmt() + .with_writer(capture.clone()) + .with_ansi(false) + .with_max_level(tracing::Level::INFO) + .finish(); + let _sub_guard = tracing::subscriber::set_default(subscriber); + + // Uploads still succeed via the dues fallback despite zero size-eligibility. + let content = Bytes::from("adr-0005 fast-growth payload"); + let address = compute_address(&content); + let stored = client + .chunk_put(content.clone()) + .await + .expect("dues fallback must keep uploads working under fast growth"); + assert_eq!(stored, address); + + let captured = String::from_utf8_lossy(&capture.0.lock().expect("capture lock")).to_string(); + + // The gate actually used the dues tier (not size, not ungated). + assert!( + captured + .lines() + .any(|l| l.contains("DUES-ELIGIBLE (size relaxed)")), + "the gate must fall back to the dues tier, not stay size or go ungated" + ); + + // Every decision for a well-behaved subject: NOT size-eligible (grown size + // uncovered) but dues-eligible. + let sized_lines: Vec<&str> = captured + .lines() + .filter(|l| l.contains("eligibility decision")) + .collect(); + assert!( + sized_lines + .iter() + .any(|l| l.contains("eligible=false") && l.contains("dues_eligible=true")), + "well-behaved nodes are dues-eligible but not size-eligible" + ); + + // The fresh node and the convicted node are NEITHER size- nor + // dues-eligible. (A single upload's close group may not include a given + // node; retry a few uploads so each is evaluated at least once, then + // assert every decision for it fails both tiers — non-vacuously.) + for extra in 0..8 { + let c = Bytes::from(format!("adr-0005 fast-growth probe {extra}")); + let _ = client.chunk_put(c).await; + } + let captured = String::from_utf8_lossy(&capture.0.lock().expect("capture lock")).to_string(); + for (label, hex) in [ + ("fresh", &fresh_peer_hex), + ("convicted", &convicted_peer_hex), + ] { + let decisions: Vec<&str> = captured + .lines() + .filter(|l| l.contains("eligibility decision") && l.contains(hex.as_str())) + .collect(); + assert!( + !decisions.is_empty(), + "the {label} node must have been evaluated at least once" + ); + assert!( + decisions + .iter() + .all(|l| l.contains("eligible=false") && l.contains("dues_eligible=false")), + "the {label} node must fail BOTH tiers even under the dues fallback" + ); + } + + testnet.teardown().await; +} diff --git a/ant-core/tests/support/mod.rs b/ant-core/tests/support/mod.rs index 9486fecc..b7fcab0e 100644 --- a/ant-core/tests/support/mod.rs +++ b/ant-core/tests/support/mod.rs @@ -24,6 +24,7 @@ use ant_node::payment::{ EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, QuoteGenerator, QuotingMetricsTracker, }; +use ant_node::replication::audit_tally::{AuditTally, TallyReportSource}; use ant_node::replication::commitment_state::{BuiltCommitment, ResponderCommitmentState}; use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; // Wire / transport / EVM types: route through ant-protocol so the test @@ -97,6 +98,12 @@ pub fn test_client_config() -> ClientConfig { pub struct TestNode { pub p2p_node: Option>, pub protocol: Option>, + /// ADR-0005: this node's audit tally (present when the testnet was + /// started with tallies). Tests seed it directly — MiniTestnet nodes run + /// no replication engine, so no real audits feed it — which makes the + /// tally -> signed report -> client verify -> eligibility gate path fully + /// deterministic. + pub audit_tally: Option>>, _handler_task: Option>, } @@ -115,7 +122,7 @@ impl MiniTestnet { /// Use `DEFAULT_NODE_COUNT` for standard tests, 35+ for merkle tests (need 16 peers per pool). /// Nodes emit baseline `(0, None)` quotes (no commitment source wired). pub async fn start(node_count: usize) -> Self { - Self::start_inner(node_count, None).await + Self::start_inner(node_count, None, false).await } /// ADR-0004: start a testnet where every node carries a live storage @@ -124,10 +131,21 @@ impl MiniTestnet { /// in the quote response). Exercises the full node→client→storer binding /// handshake against real QUIC + Anvil. pub async fn start_with_commitments(node_count: usize, key_count: u32) -> Self { - Self::start_inner(node_count, Some(key_count)).await + Self::start_inner(node_count, Some(key_count), false).await } - async fn start_inner(node_count: usize, commitment_key_count: Option) -> Self { + /// ADR-0005: like [`Self::start_with_commitments`] but every node also + /// carries an (empty) audit tally wired as its quote generator's report + /// source. Tests seed the tallies and the nodes testify accordingly. + pub async fn start_with_commitments_and_tallies(node_count: usize, key_count: u32) -> Self { + Self::start_inner(node_count, Some(key_count), true).await + } + + async fn start_inner( + node_count: usize, + commitment_key_count: Option, + with_tallies: bool, + ) -> Self { // Start Anvil EVM testnet FIRST let testnet = Testnet::new().await.expect("start Anvil testnet"); let evm_network = testnet.to_network(); @@ -152,13 +170,14 @@ impl MiniTestnet { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let (node, protocol, handler) = Self::spawn_node( + let (node, protocol, tally, handler) = Self::spawn_node( addr, &bootstrap_addrs, temp_dir.path(), &evm_network, i, commitment_key_count, + with_tallies, ) .await; @@ -166,6 +185,7 @@ impl MiniTestnet { nodes.push(TestNode { p2p_node: Some(Arc::clone(&node)), protocol: Some(protocol), + audit_tally: tally, _handler_task: Some(handler), }); temp_dirs.push(temp_dir); @@ -178,19 +198,21 @@ impl MiniTestnet { let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port); let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let (node, protocol, handler) = Self::spawn_node( + let (node, protocol, tally, handler) = Self::spawn_node( addr, &bootstrap_addrs, temp_dir.path(), &evm_network, i, commitment_key_count, + with_tallies, ) .await; nodes.push(TestNode { p2p_node: Some(Arc::clone(&node)), protocol: Some(protocol), + audit_tally: tally, _handler_task: Some(handler), }); temp_dirs.push(temp_dir); @@ -284,7 +306,7 @@ impl MiniTestnet { &self.evm_network } - #[allow(clippy::too_many_lines)] + #[allow(clippy::too_many_lines, clippy::type_complexity)] async fn spawn_node( listen_addr: SocketAddr, bootstrap_peers: &[SocketAddr], @@ -292,7 +314,13 @@ impl MiniTestnet { evm_network: &EvmNetwork, node_index: usize, commitment_key_count: Option, - ) -> (Arc, Arc, tokio::task::JoinHandle<()>) { + with_tally: bool, + ) -> ( + Arc, + Arc, + Option>>, + tokio::task::JoinHandle<()>, + ) { // Generate ML-DSA-65 identity for this node let identity = Arc::new(NodeIdentity::generate().expect("generate node identity")); @@ -383,6 +411,20 @@ impl MiniTestnet { protocol.attach_commitment_source(source); } + // ADR-0005: wire an (empty) audit tally as the report source so quote + // responses carry this node's signed testimony once a test seeds it. + let audit_tally = if with_tally { + let tally = Arc::new(std::sync::RwLock::new(AuditTally::default())); + let reporter_peer_id = *blake3::hash(identity.public_key().as_bytes()).as_bytes(); + protocol.attach_report_source(Arc::new(TallyReportSource::new( + Arc::clone(&tally), + reporter_peer_id, + ))); + Some(tally) + } else { + None + }; + // Start message handler loop let handler_node = Arc::clone(&node); let handler_protocol = Arc::clone(&protocol); @@ -440,7 +482,114 @@ impl MiniTestnet { } }); - (node, protocol, handler) + (node, protocol, audit_tally, handler) + } + + /// ADR-0005: seed `observer`'s tally with a clean audited history for + /// `subject`: one pass per day for `days` distinct days ending now, each + /// at `key_count` committed keys. `days = 7` is a full week of dues. + pub fn seed_history(&self, observer: usize, subject: &[u8; 32], key_count: u32, days: u64) { + let tally = self.nodes[observer] + .audit_tally + .as_ref() + .expect("testnet started with tallies"); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_secs(); + let mut guard = tally.write().expect("tally lock"); + let subject = ant_protocol::transport::PeerId::from_bytes(*subject); + // Three passes per day: 7 days -> 21 passes, clearing the default + // ADR5_P_PASSES = 20. + for d in 0..days { + for p in 0..3u64 { + guard.record_pass(subject, key_count, now - d * 86_400 - p); + } + } + } + + /// ADR-0005: every tally-bearing node except `subject_index` gets a clean + /// `days`-day history for node `subject_index`. + pub fn seed_history_for_all_observers(&self, subject_index: usize, key_count: u32, days: u64) { + let subject = *self.nodes[subject_index] + .p2p_node + .as_ref() + .expect("subject node running") + .peer_id() + .as_bytes(); + for (i, node) in self.nodes.iter().enumerate() { + if i != subject_index && node.audit_tally.is_some() { + self.seed_history(i, &subject, key_count, days); + } + } + } + + /// ADR-0005: record a deterministic conviction of `subject_index` at every + /// other tally-bearing node (what a real caught-deleting node would face), + /// `age_days` in the past (0 = now; v4 sticky convictions last + /// `CONVICTION_STICKY_DAYS`, so tests use an old conviction to model "the + /// sticky period has since elapsed"). + pub fn convict_at_all_observers(&self, subject_index: usize, age_days: u64) { + let subject = *self.nodes[subject_index] + .p2p_node + .as_ref() + .expect("subject node running") + .peer_id() + .as_bytes(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_secs() + - age_days * 86_400; + let subject = ant_protocol::transport::PeerId::from_bytes(subject); + for (i, node) in self.nodes.iter().enumerate() { + if i == subject_index { + continue; + } + if let Some(tally) = &node.audit_tally { + tally + .write() + .expect("tally lock") + .record_conviction(subject, now); + } + } + } + + /// ADR-0005: fence `subject_index` at every other tally-bearing node (what + /// a silent hopper faces after ignoring monetized-pin challenges). + pub fn fence_at_all_observers(&self, subject_index: usize) { + let subject = *self.nodes[subject_index] + .p2p_node + .as_ref() + .expect("subject node running") + .peer_id() + .as_bytes(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_secs(); + let subject = ant_protocol::transport::PeerId::from_bytes(subject); + for (i, node) in self.nodes.iter().enumerate() { + if i == subject_index { + continue; + } + if let Some(tally) = &node.audit_tally { + tally + .write() + .expect("tally lock") + .record_unanswered_monetized_challenge(subject, now); + } + } + } + + /// Peer id bytes of node `index`. + pub fn peer_id_bytes(&self, index: usize) -> [u8; 32] { + *self.nodes[index] + .p2p_node + .as_ref() + .expect("node running") + .peer_id() + .as_bytes() } /// Shut down a node by index, simulating a failure.