From 75de2327241ea1185caa85eaca279849b9ca1930 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 31 Aug 2026 18:19:38 +0000 Subject: [PATCH] test(usecases): self-check hotpath coverage in workload benches Co-authored-by: Zack Jackson --- .../benches/hotpath_coverage.rs | 162 ++++++++++++++++++ .../benches/semantic_vector_commit_scale.rs | 17 ++ .../benches/work_rollup.rs | 37 ++++ 3 files changed, 216 insertions(+) create mode 100644 crates/tracedecay-usecases/benches/hotpath_coverage.rs diff --git a/crates/tracedecay-usecases/benches/hotpath_coverage.rs b/crates/tracedecay-usecases/benches/hotpath_coverage.rs new file mode 100644 index 0000000000..7a7099e570 --- /dev/null +++ b/crates/tracedecay-usecases/benches/hotpath_coverage.rs @@ -0,0 +1,162 @@ +//! Shared Hotpath coverage support for this crate's bench binaries. +//! +//! The workload benches (`work_rollup`, `semantic_vector_commit_scale`) +//! compile in two modes and this module makes both modes self-checking: +//! +//! - feature off (`default`): every hotpath macro in the workload is a +//! no-op. [`init`] records the state of any operator-named +//! `HOTPATH_OUTPUT_PATH` and [`finish`] asserts the run never created or +//! modified a report there - profiling must not leak into default builds. +//! - feature on (`--features hotpath`): [`init`] forces the metrics server +//! off (these workloads are specified to open no socket) and installs the +//! process-boundary guard. When the operator did not name a report +//! destination the run self-verifies: the report goes to a scratch JSON +//! file and [`finish`] drops the guard, parses the report, and asserts the +//! expected static `crate.area.verb` labels were recorded. When the +//! operator did name `HOTPATH_OUTPUT_PATH` the profile belongs to them: +//! the guard honors their configuration and no verification synthesizes +//! extra work into their report. +//! +//! Labels passed to [`finish`] must be labels this crate already stamps +//! (`#[hotpath::measure]`, `measure_block!`, `gauge!`); this module never +//! introduces its own measurement labels. +//! +//! The environment mutation in [`init`] is sound for the same reason as in +//! `tracedecay-index-bench`: it runs as the first statement of `main`, +//! before the Tokio runtime, the guard, or any other thread exists. + +#[cfg(feature = "hotpath")] +pub struct HotpathCoverage { + bench: &'static str, + guard: Option, + verified_report: Option, +} + +#[cfg(feature = "hotpath")] +pub fn init(bench: &'static str) -> HotpathCoverage { + // Hotpath binds a localhost metrics server on guard construction. These + // benches open no socket, so the server stays off unless an operator + // explicitly asked for it. + if std::env::var_os("HOTPATH_METRICS_SERVER_OFF").is_none() { + unsafe { + std::env::set_var("HOTPATH_METRICS_SERVER_OFF", "1"); + } + } + let operator_owns_report = std::env::var_os("HOTPATH_OUTPUT_PATH") + .is_some_and(|path| path.to_str().is_some_and(|path| !path.is_empty())); + let verified_report = if operator_owns_report { + None + } else { + // Self-verified mode: the report goes to a scratch file rather than + // stdout (stdout carries each bench's machine-readable lines) and is + // parsed on `finish`. `functions-timing` and `futures` carry + // `#[hotpath::measure]` spans; `debug` carries `gauge!` keys. + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_nanos()) + .unwrap_or_default(); + let path = std::env::temp_dir().join(format!( + "{bench}-hotpath-coverage-{}-{unique:x}.json", + std::process::id() + )); + unsafe { + std::env::set_var("HOTPATH_OUTPUT_FORMAT", "json"); + std::env::set_var("HOTPATH_OUTPUT_PATH", &path); + } + if std::env::var_os("HOTPATH_REPORT").is_none() { + unsafe { + std::env::set_var("HOTPATH_REPORT", "functions-timing,futures,debug"); + } + } + Some(path) + }; + let guard = hotpath::HotpathGuardBuilder::new(bench).build(); + HotpathCoverage { + bench, + guard: Some(guard), + verified_report, + } +} + +#[cfg(feature = "hotpath")] +impl HotpathCoverage { + /// True when this run owns a scratch report and label verification will + /// run on [`finish`]. Operator-owned profiling runs return false so + /// coverage probes never synthesize work into a real profile. + #[allow(dead_code)] + pub fn verifying(&self) -> bool { + self.verified_report.is_some() + } +} + +#[cfg(feature = "hotpath")] +pub fn finish(mut coverage: HotpathCoverage, expected_labels: &[&str]) { + // Dropping the guard at this graceful boundary (never `process::exit`) + // is what emits the exit report. + drop(coverage.guard.take()); + let Some(report_path) = coverage.verified_report.take() else { + println!( + "hotpath_coverage,bench={},mode=operator_report", + coverage.bench + ); + return; + }; + let report = std::fs::read_to_string(&report_path).expect( + "a feature-on bench run must write a Hotpath JSON report when its guard drops", + ); + assert!(!report.is_empty(), "Hotpath report must not be empty"); + serde_json::from_str::(&report) + .expect("Hotpath report must be valid JSON"); + for label in expected_labels { + assert!( + report.contains(label), + "Hotpath report at {} is missing the static label {label:?}", + report_path.display(), + ); + } + std::fs::remove_file(&report_path).expect("remove scratch Hotpath report"); + println!( + "hotpath_coverage,bench={},mode=verified,labels_verified={},report_bytes={}", + coverage.bench, + expected_labels.len(), + report.len(), + ); +} + +#[cfg(not(feature = "hotpath"))] +pub struct HotpathCoverage { + bench: &'static str, + operator_report: Option<(std::path::PathBuf, Option>)>, +} + +#[cfg(not(feature = "hotpath"))] +pub fn init(bench: &'static str) -> HotpathCoverage { + // Feature off, every hotpath macro in the workload expands to a no-op + // and nothing reads the report environment. Capture the pre-run state of + // any operator-named destination so `finish` can prove that stayed true. + let operator_report = std::env::var_os("HOTPATH_OUTPUT_PATH") + .filter(|path| !path.is_empty()) + .map(std::path::PathBuf::from) + .map(|path| { + let prior = std::fs::read(&path).ok(); + (path, prior) + }); + HotpathCoverage { + bench, + operator_report, + } +} + +#[cfg(not(feature = "hotpath"))] +pub fn finish(coverage: HotpathCoverage, _expected_labels: &[&str]) { + if let Some((path, prior)) = coverage.operator_report { + let current = std::fs::read(&path).ok(); + assert_eq!( + current, + prior, + "a feature-off bench build must never write a Hotpath report to {}", + path.display(), + ); + } + println!("hotpath_coverage,bench={},mode=feature_off", coverage.bench); +} diff --git a/crates/tracedecay-usecases/benches/semantic_vector_commit_scale.rs b/crates/tracedecay-usecases/benches/semantic_vector_commit_scale.rs index 2d8cbde003..be13dc6d9c 100644 --- a/crates/tracedecay-usecases/benches/semantic_vector_commit_scale.rs +++ b/crates/tracedecay-usecases/benches/semantic_vector_commit_scale.rs @@ -23,6 +23,9 @@ use std::fmt::Write as _; use std::time::Instant; +#[path = "hotpath_coverage.rs"] +mod hotpath_coverage; + use sha2::{Digest, Sha256}; use tracedecay_code_index::projection::{ ChunkProjectionDecisionV1, build_batch_receipt, expected_request_digest, @@ -214,7 +217,20 @@ fn report(label: &str, batch: usize, rows: usize, started: &Instant) { ); } +// This workload deliberately drives the in-memory reference machine +// (`VectorGenerationStateMachineV1`), which carries no instrumentation of its +// own; the `usecases.store.*` spans and `usecases.vector.*` gauges live on +// the graph adapter, which needs a live graph runtime this bench must not +// mount. Coverage here is therefore the feature toggle itself: feature off, +// the macros are no-ops and no report may appear; feature on, the guard +// lifecycle completes with the metrics server forced off and a parseable +// exit report. Extend this list when the machine itself gains labels. +const EXPECTED_HOTPATH_LABELS: &[&str] = &[]; + fn main() { + // First statement on purpose: may set Hotpath environment for the guard, + // which is sound only before any other thread exists. + let coverage = hotpath_coverage::init("tracedecay-semantic-vector-commit-scale"); let chunks = env_scale("TD_SCALE_CHUNKS", 120_000); let dimensions = env_scale("TD_SCALE_DIMS", 768); let batch_len = env_scale("TD_SCALE_BATCH", 512); @@ -301,4 +317,5 @@ fn main() { assert_eq!(generation_rows, chunks, "published corpus is complete"); drop(state); report("dropped", batch_count, committed_rows, &started); + hotpath_coverage::finish(coverage, EXPECTED_HOTPATH_LABELS); } diff --git a/crates/tracedecay-usecases/benches/work_rollup.rs b/crates/tracedecay-usecases/benches/work_rollup.rs index 1959bc5584..d79f7162df 100644 --- a/crates/tracedecay-usecases/benches/work_rollup.rs +++ b/crates/tracedecay-usecases/benches/work_rollup.rs @@ -7,6 +7,9 @@ use std::time::Duration; #[path = "../tests/observability_runtime_contract.rs"] mod observability_runtime_contract; +#[path = "hotpath_coverage.rs"] +mod hotpath_coverage; + use observability_runtime_contract::work_rollup_harness::{ READ_TRIPWIRE, SOURCE_COUNT, TRIPWIRE, WORK_ROLLUP_BENCHMARK_ARTIFACT_SCHEMA_VERSION, WorkRollupBenchmarkArtifactV1, WorkRollupFixtureV1, WorkRollupFreshStoreMeasurementV1, @@ -91,6 +94,9 @@ fn validate_completion(report: &WorkRollupReport) { } fn main() { + // First statement on purpose: may set Hotpath environment for the guard, + // which is sound only before the runtime or any other thread exists. + let coverage = hotpath_coverage::init("tracedecay-work-rollup"); let artifact_path = explicit_artifact_path(); let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -253,4 +259,35 @@ fn main() { }; write_jsonl_artifact(&artifact_path, &artifact).expect("write Work rollup JSONL artifact"); println!("work_rollup artifact={}", artifact_path.display()); + + // Post-measurement label coverage, self-verified runs only (never during + // operator profiling): one canonical Observatory read over a fresh + // registered store traverses `usecases.observability.read_model`, the + // static span already stamped on the shared read composition this + // bench's rollup surface belongs to, so the exit report must carry it. + #[cfg(feature = "hotpath")] + if coverage.verifying() { + runtime.block_on(async { + let _pin = tracedecay_runtime_core::config::PinnedUserDataDir::new(); + let db_runtime = + tracedecay_global_db::tests::harness::RegisteredGlobalDbTestRuntime::profile( + tracedecay_runtime_core::storage::default_profile_root() + .expect("hotpath coverage profile root"), + ) + .await + .expect("hotpath coverage registered runtime"); + let database = db_runtime.profile_database_arc(); + let read_model = tracedecay_usecases::observability::observatory_read_model( + database.as_ref(), + None, + 0, + ) + .await; + assert!( + !read_model.metrics.is_empty(), + "observatory read model must project metrics" + ); + }); + } + hotpath_coverage::finish(coverage, &["usecases.observability.read_model"]); }