From 7daf9770c12dcd439f165e03947762f06dcdb121 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Sun, 13 Sep 2026 12:46:08 +0700 Subject: [PATCH 1/5] codex scanner: detect whitespace turn_context candidates (cherry picked from commit f8732f99ccae7e156d69c4d30ede15ca61a2e890) --- rust/src/core/jsonl_scanner/codex/helpers.rs | 1 + rust/src/core/jsonl_scanner/tests.rs | 43 ++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/rust/src/core/jsonl_scanner/codex/helpers.rs b/rust/src/core/jsonl_scanner/codex/helpers.rs index e559700cbb..9bd3881adb 100644 --- a/rust/src/core/jsonl_scanner/codex/helpers.rs +++ b/rust/src/core/jsonl_scanner/codex/helpers.rs @@ -323,6 +323,7 @@ pub(super) fn parse_codex_fast_event(line: &str) -> Option> { pub(super) fn is_candidate_codex_line(line: &str) -> bool { if !line.contains("\"type\":\"event_msg\"") && !line.contains("\"type\":\"turn_context\"") + && !line.contains("\"turn_context\"") && !line.contains("\"event_msg\"") { return false; diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index bcb1129cb1..a009233cdc 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -515,6 +515,49 @@ fn test_fast_codex_parser_reads_legacy_event_msg_shape() { assert_eq!((record.input, record.cached, record.output), (20, 5, 3)); } +#[test] +fn codex_fast_parser_accepts_compact_and_spaced_event_records_equally() { + let range = CostUsageDayRange::new( + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(), + ); + let parsed = |line: &str| { + let mut parser = CodexParserState::new(Some("gpt-5".to_string()), None); + parser.process_line(line, &range); + ( + parser.current_model, + parser + .records + .into_iter() + .map(|record| { + ( + record.day_key, + record.model, + record.input, + record.cached, + record.output, + record.reasoning, + ) + }) + .collect::>(), + ) + }; + + let compact_event = r#"{"timestamp":"2026-05-31T10:00:01Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":20,"cached_input_tokens":5,"output_tokens":3}}}}"#; + let spaced_event = r#"{ "timestamp": "2026-05-31T10:00:01Z", "type": "event_msg", "payload": { "type": "token_count", "info": { "last_token_usage": { "input_tokens": 20, "cached_input_tokens": 5, "output_tokens": 3 } } } }"#; + assert!(is_candidate_codex_line(spaced_event)); + assert_eq!(parsed(compact_event), parsed(spaced_event)); + + let compact_context = r#"{"timestamp":"2026-05-31T10:00:00Z","type":"turn_context","payload":{"model":"gpt-5.5"}}"#; + let spaced_context = "{\t\"timestamp\": \"2026-05-31T10:00:00Z\",\t\"type\":\t\"turn_context\",\t\"payload\": {\t\"model\": \"gpt-5.5\"\t}\t}"; + assert!(is_candidate_codex_line(spaced_context)); + assert_eq!(parsed(compact_context), parsed(spaced_context)); + + let unrelated = r#"{ "timestamp": "2026-05-31T10:00:00Z", "type": "response", "payload": { "model": "gpt-5.5" } }"#; + assert!(!is_candidate_codex_line(unrelated)); + assert_eq!(parsed(unrelated), (Some("gpt-5".to_string()), Vec::new())); +} + #[test] fn test_parse_codex_file_uses_fast_parser_for_current_logs() { let mut file = tempfile::NamedTempFile::new().expect("temp file"); From 4778b303e45d707b8963516579d471805f59ea14 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Mon, 14 Sep 2026 03:26:48 +0700 Subject: [PATCH 2/5] Preserve bare Codex usage fallback (cherry picked from commit dccacc6aab8f70680b1bd38550f8dce55629e6bc) --- rust/src/core/jsonl_scanner/codex.rs | 4 ++-- rust/src/core/jsonl_scanner/codex/parser.rs | 5 ++++- rust/src/core/jsonl_scanner/tests.rs | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/rust/src/core/jsonl_scanner/codex.rs b/rust/src/core/jsonl_scanner/codex.rs index 30f159c0ca..a6a3b59a19 100644 --- a/rust/src/core/jsonl_scanner/codex.rs +++ b/rust/src/core/jsonl_scanner/codex.rs @@ -52,8 +52,8 @@ pub(crate) fn codex_cache_stamp_schema_version(cache: &mut CostUsageCache) { #[cfg(test)] use helpers::{ CodexFastPayload, CodexFastTotals, bare_usage_totals, codex_timestamp_day_key, - codex_totals_from_fast, fast_totals_from_payload, last_usage_delta, parse_codex_timestamp, - read_token_totals, + codex_totals_from_fast, fast_totals_from_payload, is_candidate_codex_line, last_usage_delta, + parse_codex_timestamp, read_token_totals, }; impl JsonlScanner { diff --git a/rust/src/core/jsonl_scanner/codex/parser.rs b/rust/src/core/jsonl_scanner/codex/parser.rs index 2ec70f9cf1..a37a2ad721 100644 --- a/rust/src/core/jsonl_scanner/codex/parser.rs +++ b/rust/src/core/jsonl_scanner/codex/parser.rs @@ -109,7 +109,10 @@ impl CodexParserState { source_end_offset: i64, ) { let event_candidate = is_candidate_codex_line(line); - let bare_candidate = !event_candidate && line.contains("\"usage\""); + // Event candidacy is only a fast-path hint. It must not suppress the + // independent bare-usage record family when a model value happens to + // contain an event marker such as "turn_context". + let bare_candidate = line.contains("\"usage\""); if !event_candidate && !bare_candidate { return; } diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index a009233cdc..2053136417 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -949,6 +949,22 @@ fn process_line_accepts_type_less_bare_usage_row() { ); } +#[test] +fn process_line_keeps_bare_usage_when_model_contains_turn_context() { + let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); + let range = CostUsageDayRange::new(day, day); + let mut parser = CodexParserState::new(None, None); + + parser.process_line( + r#"{"timestamp":"2026-05-31T10:00:01Z","model":"turn_context","usage":{"prompt_tokens":120,"completion_tokens":30}}"#, + &range, + ); + + assert_eq!(parser.records.len(), 1); + assert_eq!(parser.records[0].input, 120); + assert_eq!(parser.records[0].output, 30); +} + #[test] fn timestamp_less_bare_usage_uses_last_accepted_usage_day() { let day = NaiveDate::from_ymd_opt(2026, 5, 31).unwrap(); From ef326cf76f3b0f7fe340e36e0716812b64a60afa Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:11:30 +0700 Subject: [PATCH 3/5] Port v0.57.0 OpenCodex numeric safety (cherry picked from commit 399741e3ac1deace8cfbd4ba09b8579998aaf060) --- rust/src/spend_contract/opencodex.rs | 19 ++++-- rust/src/spend_contract/opencodex/cache.rs | 78 +++++++++++++++++++++- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/rust/src/spend_contract/opencodex.rs b/rust/src/spend_contract/opencodex.rs index 6649e07e87..229dc34e74 100644 --- a/rust/src/spend_contract/opencodex.rs +++ b/rust/src/spend_contract/opencodex.rs @@ -498,13 +498,14 @@ fn nonnegative_u64(value: Option<&Value>) -> Option { return Some(number); } let number = value.as_f64()?; - // Guarded to finite non-negative values within u64 range. + // `u64::MAX as f64` rounds to 2^64, so the exclusive bound rejects that + // unrepresentable floating-point boundary before the saturating cast. #[allow( clippy::cast_possible_truncation, - reason = "guarded to finite non-negative values within u64 range" + reason = "finite non-negative floats below 2^64 fit the intended truncating cast" )] let parsed = - (number.is_finite() && number >= 0.0 && number <= u64::MAX as f64).then_some(number as u64); + (number.is_finite() && number >= 0.0 && number < u64::MAX as f64).then_some(number as u64); parsed } @@ -963,15 +964,23 @@ mod tests { #[test] fn nonnegative_u64_accepts_json_numbers_and_bounded_floats() { assert_eq!(nonnegative_u64(Some(&serde_json::json!(42))), Some(42)); + assert_eq!( + nonnegative_u64(Some(&serde_json::json!(u64::MAX))), + Some(u64::MAX) + ); assert_eq!(nonnegative_u64(Some(&serde_json::json!(12.0))), Some(12)); // Fractional floats are accepted via `as u64` truncation. assert_eq!(nonnegative_u64(Some(&serde_json::json!(1.5))), Some(1)); - // `u64::MAX as f64` rounds up to 2^64; f64 spacing there is 4096, so - // +2048.0 rounds back into range. First out-of-range step is +4096.0. + // `u64::MAX as f64` is the unrepresentable 2^64 boundary. + assert_eq!( + nonnegative_u64(Some(&serde_json::json!(u64::MAX as f64))), + None + ); assert_eq!( nonnegative_u64(Some(&serde_json::json!(u64::MAX as f64 + 4096.0))), None ); + assert_eq!(nonnegative_u64(Some(&serde_json::json!(-1))), None); assert_eq!(nonnegative_u64(None), None); } } diff --git a/rust/src/spend_contract/opencodex/cache.rs b/rust/src/spend_contract/opencodex/cache.rs index 73fca32e72..a8598e141a 100644 --- a/rust/src/spend_contract/opencodex/cache.rs +++ b/rust/src/spend_contract/opencodex/cache.rs @@ -10,6 +10,7 @@ use sha2::{Digest, Sha256}; use super::{OpenCodexEntry, parse_line}; const CACHE_SCHEMA_VERSION: i64 = 2; +const PARSER_VERSION: u32 = 1; const PREFIX_DIGEST_BYTES: u64 = 64 * 1024; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -18,6 +19,8 @@ pub(super) struct ParseCursor { file_identity: String, pub(super) parsed_offset: u64, prefix_digest: String, + #[serde(default)] + parser_version: Option, } #[derive(Debug, Clone)] @@ -77,6 +80,7 @@ pub(super) fn load_entries_with_cache( file_identity: identity.file_identity.clone(), parsed_offset: parsed.next_offset, prefix_digest: prefix_digest(source_path, parsed.next_offset)?, + parser_version: Some(PARSER_VERSION), }; if !source_matches_snapshot(source_path, &identity) { continue; @@ -102,6 +106,7 @@ pub(super) fn load_entries_with_cache( file_identity: identity.file_identity.clone(), parsed_offset: parsed.next_offset, prefix_digest: prefix_digest(source_path, parsed.next_offset)?, + parser_version: Some(PARSER_VERSION), }; if !source_matches_snapshot(source_path, &identity) { continue; @@ -157,7 +162,8 @@ fn source_matches_snapshot(source_path: &Path, identity: &LogIdentity) -> bool { } fn cursor_matches_source(cursor: &ParseCursor, identity: &LogIdentity, source_path: &Path) -> bool { - cursor.source_path == identity.source_path + cursor.parser_version == Some(PARSER_VERSION) + && cursor.source_path == identity.source_path && cursor.file_identity == identity.file_identity && identity.size >= cursor.parsed_offset && prefix_digest(source_path, cursor.parsed_offset) @@ -384,6 +390,76 @@ fn platform_file_identity(_source_path: &Path, metadata: &fs::Metadata) -> Optio mod tests { use super::*; + fn write_test_log(path: &Path) { + std::fs::write( + path, + br#"{"requestId":"entry","model":"gpt-5","timestamp":"2026-08-18T10:00:00Z","usageStatus":"reported","usage":{"inputTokens":1}} +"#, + ) + .expect("write log"); + } + + #[test] + fn parser_version_gate_reparses_legacy_and_mismatched_cursors() { + let dir = tempfile::tempdir().expect("tempdir"); + let log_path = dir.path().join("usage.jsonl"); + let cache_path = dir.path().join("cache.sqlite"); + write_test_log(&log_path); + + let entries = load_entries_with_cache(&log_path, &cache_path).expect("initial load"); + assert_eq!(entries.len(), 1); + let current = read_cache(&cache_path).expect("current cache"); + let identity = log_identity(&log_path).expect("log identity"); + assert_eq!(current.cursor.parser_version, Some(PARSER_VERSION)); + assert!(cursor_matches_source(¤t.cursor, &identity, &log_path)); + let encoded = serde_json::to_value(¤t.cursor).expect("encode cursor"); + assert_eq!( + encoded + .get("parser_version") + .and_then(serde_json::Value::as_u64), + Some(u64::from(PARSER_VERSION)) + ); + + for parser_version in [None, Some(0), Some(2)] { + let mut candidate = current.cursor.clone(); + candidate.parser_version = parser_version; + assert!(!cursor_matches_source(&candidate, &identity, &log_path)); + } + + let conn = Connection::open(&cache_path).expect("open cache"); + let cursor_json: String = conn + .query_row( + "SELECT value FROM meta WHERE key = 'parseCursor'", + [], + |row| row.get(0), + ) + .expect("read cursor"); + let mut legacy: serde_json::Value = + serde_json::from_str(&cursor_json).expect("decode cursor"); + legacy + .as_object_mut() + .expect("cursor object") + .remove("parser_version"); + conn.execute( + "UPDATE meta SET value = ?1 WHERE key = 'parseCursor'", + params![serde_json::to_string(&legacy).expect("encode legacy cursor")], + ) + .expect("write legacy cursor"); + drop(conn); + + let legacy_state = read_cache(&cache_path).expect("load legacy cache"); + assert_eq!(legacy_state.cursor.parser_version, None); + let reparsed = load_entries_with_cache(&log_path, &cache_path).expect("reparse legacy"); + assert_eq!(reparsed.len(), 1); + assert_eq!( + read_cache(&cache_path) + .expect("migrated cache") + .cursor + .parser_version, + Some(PARSER_VERSION) + ); + } + #[test] fn same_path_replacement_forces_reparse() { let dir = tempfile::tempdir().expect("tempdir"); From a3a7bec936268405cedeee8e1a73de5f0ba49378 Mon Sep 17 00:00:00 2001 From: NessZerra <90105158+Finesssee@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:34:52 +0700 Subject: [PATCH 4/5] Port v0.57.0: map gpt-reserve pricing alias to gpt-5.6-luna (cherry picked from commit ccd5867ed7661e1f784e63869aa4640fd0bbdfdc) --- rust/src/core/cost_pricing.rs | 11 ++++++-- rust/src/core/cost_pricing_tests.rs | 40 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/rust/src/core/cost_pricing.rs b/rust/src/core/cost_pricing.rs index 461b6c2ff2..6c73c9d130 100755 --- a/rust/src/core/cost_pricing.rs +++ b/rust/src/core/cost_pricing.rs @@ -624,11 +624,18 @@ impl CostUsagePricing { return Self::CODEX_UNATTRIBUTED_MODEL.to_string(); } - // Remove "openai/" prefix - if let Some(rest) = trimmed.strip_prefix("openai/") { + // Remove the provider-qualified OpenAI prefix. + if let Some((prefix, rest)) = trimmed.split_once('/') + && prefix.eq_ignore_ascii_case("openai") + { trimmed = rest.to_string(); } + // Codex uses this alias for the Luna reserve quota bucket. + if trimmed.eq_ignore_ascii_case("gpt-reserve") { + return "gpt-5.6-luna".to_string(); + } + // Check if base model (without -codex suffix) exists in pricing if let Some(idx) = trimmed.find("-codex") { let base = &trimmed[..idx]; diff --git a/rust/src/core/cost_pricing_tests.rs b/rust/src/core/cost_pricing_tests.rs index 9984061644..e3ae70328a 100644 --- a/rust/src/core/cost_pricing_tests.rs +++ b/rust/src/core/cost_pricing_tests.rs @@ -20,6 +20,30 @@ fn test_normalize_codex_model() { CostUsagePricing::normalize_codex_model("unknown"), CostUsagePricing::CODEX_UNATTRIBUTED_MODEL ); + assert_eq!( + CostUsagePricing::normalize_codex_model("gpt-reserve"), + "gpt-5.6-luna" + ); + assert_eq!( + CostUsagePricing::normalize_codex_model(" GPT-RESERVE "), + "gpt-5.6-luna" + ); + assert_eq!( + CostUsagePricing::normalize_codex_model("openai/gpt-reserve"), + "gpt-5.6-luna" + ); + assert_eq!( + CostUsagePricing::normalize_codex_model("OPENAI/GPT-RESERVE"), + "gpt-5.6-luna" + ); + assert_eq!( + CostUsagePricing::normalize_codex_model("gpt-reserve-preview"), + "gpt-reserve-preview" + ); + assert_eq!( + CostUsagePricing::normalize_codex_model("my-gpt-reserve"), + "my-gpt-reserve" + ); } #[test] @@ -168,6 +192,22 @@ fn test_gpt56_standard_pricing() { } } +#[test] +fn gpt_reserve_alias_uses_luna_pricing() { + let luna = + CostUsagePricing::codex_cost_usd("gpt-5.6-luna", 1_000, 400, 1_000).expect("Luna pricing"); + for model in [ + "gpt-reserve", + " GPT-RESERVE ", + "openai/gpt-reserve", + "OPENAI/GPT-RESERVE", + ] { + let reserve = + CostUsagePricing::codex_cost_usd(model, 1_000, 400, 1_000).expect("reserve pricing"); + assert!((reserve - luna).abs() < 1e-10, "{model}"); + } +} + #[test] fn test_gpt56_long_context_pricing() { for (model, expected) in [ From 195063e86c2cb9590230a8a194fb2b812180495a Mon Sep 17 00:00:00 2001 From: RCD <90105158+Finesssee@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:29:05 +0700 Subject: [PATCH 5/5] Adapt re-landed Codex scanner tests to record tuples --- rust/src/core/jsonl_scanner/tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/src/core/jsonl_scanner/tests.rs b/rust/src/core/jsonl_scanner/tests.rs index 2053136417..58407de6f7 100644 --- a/rust/src/core/jsonl_scanner/tests.rs +++ b/rust/src/core/jsonl_scanner/tests.rs @@ -529,7 +529,7 @@ fn codex_fast_parser_accepts_compact_and_spaced_event_records_equally() { parser .records .into_iter() - .map(|record| { + .map(|(record, _)| { ( record.day_key, record.model, @@ -961,8 +961,8 @@ fn process_line_keeps_bare_usage_when_model_contains_turn_context() { ); assert_eq!(parser.records.len(), 1); - assert_eq!(parser.records[0].input, 120); - assert_eq!(parser.records[0].output, 30); + assert_eq!(parser.records[0].0.input, 120); + assert_eq!(parser.records[0].0.output, 30); } #[test]