diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 2f4329574..279a7467e 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -29,6 +29,7 @@ use trusted_server_core::settings::Settings; use trusted_server_core::settings_data::{ default_config_key, default_config_store_name, get_settings_from_config_store, }; +use trusted_server_core::trace_cookie::handle_trace_mode; use trusted_server_core::platform::RuntimeServices; @@ -255,6 +256,7 @@ enum NamedRouteHandler { /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, + TraceMode, Auction, PageBids, FirstPartyProxy, @@ -279,7 +281,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 12] { +fn named_routes() -> [NamedRoute; 13] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -320,6 +322,11 @@ fn named_routes() -> [NamedRoute; 12] { primary_methods: LEGACY_ADMIN_DENY_METHODS, handler: NamedRouteHandler::LegacyAdminDenied, }, + NamedRoute { + path: "/_ts/trace", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::TraceMode, + }, NamedRoute { path: "/auction", primary_methods: &[Method::POST], @@ -389,6 +396,9 @@ fn named_route_handler( Ok(resp) } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), + NamedRouteHandler::TraceMode => { + handle_trace_mode(&state.settings, req.uri().query()) + } NamedRouteHandler::Auction => { // Build the geo-aware EC context so the auction consent // gate sees the caller's jurisdiction — `EcContext::default()` diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index c931360f6..1686ebbb9 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -28,6 +28,7 @@ use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; use trusted_server_core::settings::Settings; +use trusted_server_core::trace_cookie::handle_trace_mode; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::build_runtime_services; @@ -461,6 +462,15 @@ fn build_router(state: &Arc) -> RouterService { .post("/_ts/admin/keys/deactivate", |_ctx: RequestContext| async { Ok::(admin_key_management_not_supported()) }) + // Render-trace toggle: arms/disarms the ts-trace cookie and + // redirects to `/`. Gated by [debug] trace_route_enabled (404 when + // off). + .get( + "/_ts/trace", + make_handler(Arc::clone(&state), |s, _services, req| async move { + handle_trace_mode(&s.settings, req.uri().query()) + }), + ) .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ae4ca421f..713e2d275 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -26,6 +26,7 @@ //! | GET | `/_ts/api/v1/identify` | [`handle_identify`] | //! | GET | `/_ts/set-tester` | [`handle_set_tester`] | //! | GET | `/_ts/clear-tester` | [`handle_clear_tester`] | +//! | GET | `/_ts/trace` | [`handle_trace_mode`] | //! | OPTIONS | `/_ts/api/v1/identify` | [`cors_preflight_identify`] | //! | POST | `/auction` | [`handle_auction`] | //! | GET | `/first-party/proxy` | [`handle_first_party_proxy`] | @@ -128,6 +129,7 @@ use trusted_server_core::settings_data::{ default_config_key, default_config_store_name, get_settings_from_config_store, }; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; +use trusted_server_core::trace_cookie::handle_trace_mode; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -587,6 +589,7 @@ async fn run_named_route( } NamedRouteHandler::SetTester => handle_set_tester(&state.settings), NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), + NamedRouteHandler::TraceMode => handle_trace_mode(&state.settings, req.uri().query()), NamedRouteHandler::Auction => { // The auction reads consent data, so the consent KV store must be // available — fail closed with 503 when it is configured but @@ -992,6 +995,7 @@ enum NamedRouteHandler { Identify, SetTester, ClearTester, + TraceMode, Auction, PageBids, FirstPartyProxy, @@ -1073,6 +1077,11 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::GET], handler: NamedRouteHandler::ClearTester, }, + NamedRoute { + path: "/_ts/trace", + primary_methods: &[Method::GET], + handler: NamedRouteHandler::TraceMode, + }, NamedRoute { path: "/auction", primary_methods: &[Method::POST], @@ -1744,6 +1753,55 @@ mod tests { ); } + #[test] + fn dispatch_trace_route_is_disabled_by_default() { + let router = test_router(); + let response = route(&router, empty_request(Method::GET, "/_ts/trace")); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "disabled trace route should return 404" + ); + assert!( + response.headers().get(header::SET_COOKIE).is_none(), + "disabled trace route should not set a cookie" + ); + } + + #[test] + fn dispatch_trace_route_arms_cookie_and_redirects() { + let mut settings = test_settings(); + settings.debug.trace_route_enabled = true; + let state = app_state_for_settings(settings); + let router = TrustedServerApp::routes_for_state(&state); + let response = route(&router, empty_request(Method::GET, "/_ts/trace")); + + assert_eq!( + response.status(), + StatusCode::FOUND, + "enabled trace route should redirect to root" + ); + assert_eq!( + response + .headers() + .get(header::LOCATION) + .and_then(|v| v.to_str().ok()), + Some("/"), + "trace route should redirect to /" + ); + let set_cookie = response + .headers() + .get(header::SET_COOKIE) + .expect("should set trace cookie") + .to_str() + .expect("should render set-cookie as utf-8"); + assert!( + set_cookie.starts_with("ts-trace=1;"), + "trace route should arm the ts-trace cookie" + ); + } + #[test] fn dispatch_set_tester_sets_cookie_on_configured_domain() { let mut settings = test_settings(); diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs index e812b95bd..f2ff5d9e5 100644 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ b/crates/trusted-server-adapter-fastly/src/backend.rs @@ -1,3 +1,4 @@ +use core::fmt::Write as _; use std::time::Duration; use error_stack::{Report, ResultExt as _}; @@ -69,11 +70,11 @@ fn spec_digest_hex(canonical: &str) -> String { let mut hasher = Sha256::new(); hasher.update(canonical.as_bytes()); let digest = hasher.finalize(); - digest - .iter() - .take(SPEC_DIGEST_HEX_LEN / 2) - .map(|byte| format!("{byte:02x}")) - .collect() + let mut hex = String::with_capacity(SPEC_DIGEST_HEX_LEN); + for byte in digest.iter().take(SPEC_DIGEST_HEX_LEN / 2) { + write!(hex, "{byte:02x}").expect("should write hex digit to string"); + } + hex } /// Default first-byte timeout for backends (15 seconds). @@ -327,11 +328,10 @@ impl<'a> BackendConfig<'a> { /// Ensure a dynamic backend exists for this configuration and return its name. /// - /// The backend name is derived from the scheme, host, port, certificate - /// setting, `first_byte_timeout`, and `between_bytes_timeout` to avoid - /// collisions. Different timeout values produce different backend - /// registrations so that a tight deadline cannot be silently widened by an - /// earlier registration. + /// The name is a collision-resistant function of the complete backend spec + /// (see `Self::compute_name`), so different specs — for example, different + /// timeout values — always produce different backend registrations and a + /// tight deadline cannot be silently widened by an earlier registration. /// /// # Errors /// diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 80f807ab2..52e9db285 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -196,6 +196,13 @@ const SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; /// the greatest rung no larger than the remaining budget, and anything above /// the top rung clamps to it. Rounding down never extends a transport cap past /// the remaining budget. +/// +/// The rung spacing trades transport window for cardinality: just below a rung +/// the haircut approaches the gap to the rung beneath (worst case ~50%, e.g. a +/// remaining budget of 9,999 ms snaps to 5,000 ms). This is accepted — on the +/// mediator path this value is the effective bound, but a denser ladder would +/// buy back at most half a bucket of transport time at the cost of +/// proportionally more backend names. const TRANSPORT_TIMEOUT_COARSE_LADDER_MS: [u32; 8] = [2000, 3000, 5000, 10000, 20000, 30000, 45000, 60000]; @@ -1130,6 +1137,12 @@ mod tests { "canonical value {value}ms (from remaining {remaining}ms) is neither a quantum \ multiple nor a ladder rung" ); + // The mediator hold bound relies on canonicalization never + // extending a transport cap past the wall-clock budget. + assert!( + value <= remaining, + "canonical value {value}ms must not extend past the remaining {remaining}ms budget" + ); } assert!( distinct.len() <= 16, @@ -1160,6 +1173,12 @@ mod tests { "canonical value {value}ms (from remaining {remaining}ms) is neither a quantum \ multiple nor a ladder rung" ); + // The mediator hold bound relies on canonicalization never + // extending a transport cap past the wall-clock budget. + assert!( + value <= remaining, + "canonical value {value}ms must not extend past the remaining {remaining}ms budget" + ); } // A budget above the top coarse rung must clamp to it, not open a new // bucket per 250ms step. diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 2291fce74..4ebab07c2 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -27,6 +27,7 @@ use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; use trusted_server_core::settings::Settings; +use trusted_server_core::trace_cookie::handle_trace_mode; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware, NormalizeMiddleware}; use crate::platform::build_runtime_services; @@ -141,7 +142,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 13] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), @@ -149,6 +150,7 @@ fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { ("/_ts/admin/keys/deactivate", &[Method::POST]), ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), + ("/_ts/trace", &[Method::GET]), ("/auction", &[Method::POST]), ("/__ts/page-bids", &[Method::GET, Method::OPTIONS]), ("/first-party/proxy", &[Method::GET]), @@ -541,6 +543,21 @@ fn build_router(state: &Arc) -> RouterService { } }; + // GET /_ts/trace — render-trace toggle: arms/disarms the ts-trace + // cookie and redirects to `/`. Gated by [debug] trace_route_enabled + // (404 when off). + let s = Arc::clone(&state); + let trace_mode_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + let req = ctx.into_request(); + Ok::( + handle_trace_mode(&s.settings, req.uri().query()) + .unwrap_or_else(|e| http_error(&e)), + ) + } + }; + // GET /__ts/page-bids — SPA re-auction endpoint. let s = Arc::clone(&state); let page_bids_handler = move |ctx: RequestContext| { @@ -730,6 +747,7 @@ fn build_router(state: &Arc) -> RouterService { // credentials and key-management payloads to the origin. .post("/_ts/admin/keys/rotate", admin_not_supported_handler) .post("/_ts/admin/keys/deactivate", admin_not_supported_handler) + .get("/_ts/trace", trace_mode_handler) .post("/auction", auction_handler) .get("/__ts/page-bids", page_bids_handler) .route( diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 8e4ad6c25..f9debc3f3 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -250,6 +250,20 @@ impl AuctionOrchestrator { } } + // A provider that is also the mediator would be called twice per + // auction — once in the bidding phase and again as the mediator. The + // mediator's own demand already flows through its mediation response, + // so the overlap is never a legitimate configuration. + if let Some(mediator_name) = &self.config.mediator + && seen.contains(mediator_name.as_str()) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Auction mediator `{mediator_name}` is also listed in [auction].providers; a provider may not mediate its own auction" + ), + })); + } + for provider_name in self .config .providers @@ -1052,17 +1066,35 @@ impl AuctionOrchestrator { let start_time = Instant::now(); match provider.request_bids(request, &provider_context).await { Ok(pending) => { - // Post-launch defense: a backend name already claimed by - // another provider this auction would misattribute the - // collected response, so fail this launch attributably - // rather than overwrite the mapping. - if backend_to_provider.contains_key(&backend_name) { + // Correlate on the *resolved* backend name the pending + // request carries, falling back to the prediction only when + // the adapter left it unset. Collection keys on the name the + // response reports (Fastly's `get_backend_name()`), so keying + // the map on the prediction here would drop a response whose + // resolved name diverged. Mirrors run_providers_parallel. + let request_backend_name = pending + .backend_name() + .map(str::to_string) + .unwrap_or_else(|| { + log::warn!( + "Provider '{}' pending request returned no backend name; \ + using predicted name '{}'", + provider.provider_name(), + backend_name, + ); + backend_name.clone() + }); + // Post-launch defense: a resolved backend name already claimed + // by another provider this auction would misattribute the + // collected response, so fail this launch attributably rather + // than overwrite the mapping. + if backend_to_provider.contains_key(&request_backend_name) { let response_time_ms = start_time.elapsed().as_millis() as u64; log::warn!( "Provider '{}' resolved to backend name '{}' already claimed by another \ provider this auction; skipping dispatch to avoid response misattribution", provider.provider_name(), - backend_name, + request_backend_name, ); launch_responses.push(provider_launch_failed_response( provider.provider_name(), @@ -1072,11 +1104,11 @@ impl AuctionOrchestrator { log::info!( "Dispatching bid request to '{}' (backend: {}, budget: {}ms)", provider.provider_name(), - backend_name, + request_backend_name, effective_timeout ); backend_to_provider.insert( - backend_name.clone(), + request_backend_name.clone(), ( provider.provider_name().to_string(), start_time, @@ -1084,7 +1116,7 @@ impl AuctionOrchestrator { effective_timeout, ), ); - pending_requests.push(pending.with_backend_name(backend_name)); + pending_requests.push(pending.with_backend_name(request_backend_name)); } } Err(e) => { @@ -1656,6 +1688,77 @@ mod tests { } } + /// Provider whose `backend_name` prediction deliberately differs from the + /// backend name its `request_bids` puts on the wire (the resolved name the + /// pending request carries and collection correlates by). Models an adapter + /// that resolves a name unequal to the orchestrator's prediction, exercising + /// the dispatched path's resolved-name correlation and post-launch guard. + struct DivergentBackendProvider { + name: &'static str, + predicted: &'static str, + resolved: &'static str, + } + + impl DivergentBackendProvider { + fn new(name: &'static str, predicted: &'static str, resolved: &'static str) -> Self { + Self { + name, + predicted, + resolved, + } + } + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for DivergentBackendProvider { + fn provider_name(&self) -> &'static str { + self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let req = PlatformHttpRequest::new( + http::Request::builder() + .method("POST") + .uri("https://example.com/bid") + .body(edgezero_core::body::Body::empty()) + .expect("should build stub bid request"), + self.resolved, + ); + context + .services + .http_client() + .send_async(req) + .await + .change_context(TrustedServerError::Auction { + message: "stub launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.name, + vec![], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + 2000 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some(self.predicted.to_string()) + } + } + /// Mediator whose context-aware parse restores `nurl`/`ad_id` (mirroring /// `adserver_mock`), while its context-free parse does not. Lets a test prove /// the synchronous mediation path calls `parse_response_with_context`. @@ -2203,6 +2306,29 @@ mod tests { ); } + #[test] + fn rejects_mediator_also_listed_as_provider() { + // A provider acting as its own mediator would be called twice per + // auction (bidding phase, then mediation); its demand already flows + // through the mediation response, so reject the overlap at startup. + let config = AuctionConfig { + enabled: true, + providers: vec!["prebid".to_string()], + mediator: Some("prebid".to_string()), + timeout_ms: 2000, + ..Default::default() + }; + let orchestrator = AuctionOrchestrator::new(config); + + let err = orchestrator + .validate_configured_provider_names() + .expect_err("should reject a mediator that is also a provider"); + assert!( + err.to_string().contains("may not mediate its own auction"), + "should explain the mediator/provider overlap, got: {err}" + ); + } + #[test] fn test_orchestrator_is_enabled() { let config = AuctionConfig { @@ -2767,6 +2893,152 @@ mod tests { }); } + #[test] + fn dispatched_resolved_backend_name_diverging_from_prediction_still_correlates() { + futures::executor::block_on(async { + // The dispatched path must correlate on the backend name the pending + // request resolves to, not the orchestrator's prediction. A provider + // whose resolved name differs from its prediction would otherwise land + // its response in the "unknown backend" branch and drop the bid. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let config = AuctionConfig { + enabled: true, + providers: vec!["provider-a".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DivergentBackendProvider::new( + "provider-a", + "predicted-backend", + "resolved-backend", + ))); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 2000, + provider_responses: None, + services, + }; + + let dispatched = match orchestrator.dispatch_auction(&request, &context).await { + DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, + _ => panic!("should dispatch the diverging provider"), + }; + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &context) + .await; + + let provider_a = result + .provider_responses + .iter() + .find(|r| r.provider == "provider-a") + .expect("should have provider-a response"); + assert_eq!( + provider_a.status, + BidStatus::Success, + "response should correlate by the resolved backend name, not the prediction" + ); + }); + } + + #[test] + fn dispatched_post_launch_resolved_name_collision_fails_second_provider_attributably() { + futures::executor::block_on(async { + // Two providers with *distinct* predictions that resolve to the SAME + // backend name. The pre-launch guard keys on the predictions and lets + // both through; only the post-launch guard, reading the resolved name, + // can catch the collision. The second must fail attributably. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); // provider-a send_async + stub.push_response(200, b"{}".to_vec()); // provider-b send_async (dropped after guard) + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let config = AuctionConfig { + enabled: true, + providers: vec!["provider-a".to_string(), "provider-b".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DivergentBackendProvider::new( + "provider-a", + "predicted-a", + "shared-resolved", + ))); + orchestrator.register_provider(Arc::new(DivergentBackendProvider::new( + "provider-b", + "predicted-b", + "shared-resolved", + ))); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 2000, + provider_responses: None, + services, + }; + + let dispatched = match orchestrator.dispatch_auction(&request, &context).await { + DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, + _ => { + panic!("should dispatch the first provider despite the resolved-name collision") + } + }; + let result = orchestrator + .collect_dispatched_auction(dispatched, services, &context) + .await; + + let provider_a = result + .provider_responses + .iter() + .find(|r| r.provider == "provider-a") + .expect("should have provider-a response"); + let provider_b = result + .provider_responses + .iter() + .find(|r| r.provider == "provider-b") + .expect("should have provider-b response"); + assert_eq!( + provider_a.status, + BidStatus::Success, + "the first provider on the shared resolved name should launch and succeed" + ); + assert_eq!( + provider_b.status, + BidStatus::Error, + "the second provider colliding on the resolved name should fail attributably" + ); + }); + } + #[test] fn select_error_is_attributed_to_correct_provider() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 31f4a6341..9654f73e6 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -249,7 +249,16 @@ pub struct Bid { pub nurl: Option, /// Billing notification URL pub burl: Option, - /// `OpenRTB` bid identifier. + /// Bid ID from the bidder (`OpenRTB` `bid.id`). + /// + /// Distinct from [`Bid::ad_id`] (the `adid` creative/ad identifier): + /// this is the bidder's identifier for the bid itself. Carried so the + /// `/auction` response can echo the upstream bid ID instead of a + /// synthesized one. Unique per bid instance, not a creative identifier — + /// always present per spec. Also used as the last-resort `hb_adid` + /// fallback in `build_bid_map` for bidders that return neither a Prebid + /// Cache UUID nor `adid`. + #[serde(default, skip_serializing_if = "Option::is_none")] pub bid_id: Option, /// Ad ID from the bidder. pub ad_id: Option, diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index ffcf4f034..6a220498d 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -5,6 +5,7 @@ pub const COOKIE_TS_EC: &str = "ts-ec"; /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; pub const COOKIE_TS_TESTER: &str = "ts-tester"; +pub const COOKIE_TS_TRACE: &str = "ts-trace"; pub const COOKIE_SHAREDID: &str = "sharedId"; pub const HEADER_X_PUB_USER_ID: HeaderName = HeaderName::from_static("x-pub-user-id"); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index e50717fb9..72a57a5ac 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1041,6 +1041,18 @@ impl IntegrationHeadInjector for PrebidIntegration { } } +/// Returns `true` when `params` is not a usable PBS bidder-params object — an +/// empty object `{}` or any non-object value such as `null`. +/// +/// PBS rejects an imp whose `bidder.` carries such a value, and it cannot +/// tell a fabricated empty from an explicitly supplied one — they are identical +/// bytes on the wire. The merge uses this to stop an unusable value from +/// clobbering real params, and the final pass uses it to drop whatever remains. +fn is_unusable_bidder_params(params: &Json) -> bool { + // `None` covers non-object values (e.g. `null`); an empty map covers `{}`. + params.as_object().is_none_or(serde_json::Map::is_empty) +} + fn expand_trusted_server_bidders( configured_bidders: &[String], params: &Json, @@ -1531,18 +1543,20 @@ impl PrebidAuctionProvider { .and_then(|p| p.get(ZONE_KEY)) .and_then(Json::as_str); - // Build the bidder map for PBS. - // The JS adapter sends "trustedServer" as the bidder (our orchestrator - // adapter name). Replace it with the real PBS bidders from config. - // Only pass through keys that are known PBS bidders — skip provider-specific - // keys like "aps" which belong to their own separate auction provider. - let mut bidder: HashMap = HashMap::new(); - // Bidders the publisher explicitly supplied — including an - // explicit empty `{}`. A configured bidder that exists only - // because `expand_trusted_server_bidders` fabricated an empty - // params object is NOT explicit and must not ship as - // `"bidder": {}` (which PBS rejects). - let mut explicit_bidders: HashSet = HashSet::new(); + // Build the bidder map for PBS from two sources: + // 1. the `trustedServer` orchestrator entry, expanded into the + // configured PBS bidders (`expand_trusted_server_bidders`); + // 2. direct entries naming a configured PBS bidder. + // The JS adapter sends "trustedServer" as the bidder (our + // orchestrator adapter name); provider-specific keys like "aps" + // run their own auction and are skipped here. + // + // The two sources are collected separately so the merge below is + // deterministic regardless of `slot.bidders` HashMap iteration + // order: direct entries win, but an unusable direct value (`{}` or + // a non-object) must not clobber real params from the expansion. + let mut expanded: HashMap = HashMap::new(); + let mut direct: Vec<(String, Json)> = Vec::new(); for (name, params) in &slot.bidders { if name == "aps" { // Trusted Server APS is a separate OpenRTB provider. Never @@ -1550,18 +1564,13 @@ impl PrebidAuctionProvider { continue; } if name == TRUSTED_SERVER_BIDDER { - if let Some(per_bidder) = - params.get(BIDDER_PARAMS_KEY).and_then(Json::as_object) - { - explicit_bidders.extend(per_bidder.keys().cloned()); - } - bidder.extend(expand_trusted_server_bidders(&self.config.bidders, params)); - bidder.remove("aps"); + expanded.extend(expand_trusted_server_bidders(&self.config.bidders, params)); + expanded.remove("aps"); } else if self.config.bidders.iter().any(|b| b == name) { - explicit_bidders.insert(name.clone()); - bidder.insert(name.clone(), params.clone()); - } else { - // Any other unrecognized key is likely a misconfiguration (a + direct.push((name.clone(), params.clone())); + } else if name != "aps" { + // `aps` is intentionally handled by its own provider. Any + // other unrecognized key is likely a misconfiguration (a // slot bidder absent from `config.bidders`) that silently // yields an empty bidder map and a stored-request no-bid — // log it so the drop is diagnosable. @@ -1573,6 +1582,28 @@ impl PrebidAuctionProvider { } } + let mut bidder = expanded; + // Bidders whose params came from the slot itself rather than from + // `trustedServer` expansion. Only these can represent a publisher + // misconfiguration if they are dropped below; a dropped fabricated + // entry is the designed stored-request path. + let mut slot_supplied: HashSet = HashSet::new(); + for (name, params) in direct { + // Direct entries win over the expansion — except when an + // unusable direct value (`{}` or a non-object) would overwrite + // real params the expansion already supplied. PBS cannot tell a + // fabricated empty from an explicit one, so real params must not + // be discarded in favour of either. + let clobbers_real = is_unusable_bidder_params(¶ms) + && bidder + .get(&name) + .is_some_and(|existing| !is_unusable_bidder_params(existing)); + if !clobbers_real { + slot_supplied.insert(name.clone()); + bidder.insert(name, params); + } + } + // Apply canonical and compatibility-derived rules in normalized // order. An override rule can populate a bidder that arrived with // empty params, promoting a fabricated empty into a valid bidder. @@ -1581,24 +1612,66 @@ impl PrebidAuctionProvider { .apply(BidParamOverrideFacts { bidder: name, zone }, params); } - // Drop bidders that are still an empty object after overrides and - // were not explicitly supplied. Shipping `"bidder": {}` makes PBS - // reject the imp; an explicit empty object is preserved so genuine - // publisher misconfiguration stays visible. + // Drop any bidder whose params are still unusable after overrides — + // an empty `{}` or a non-object like `null`. Both are byte-identical + // to PBS regardless of whether they were fabricated or supplied + // explicitly, and shipping either gets the imp rejected; a single + // non-2xx then collapses into an auction-wide bid wipeout (see + // `parse_response`), so one misconfigured slot must never reach the + // wire. + // + // The two drop sources get different levels, one line per slot. A + // slot that supplied unusable params inline is a publisher mistake + // an operator can act on, so it warns — the PBS 400 body is + // otherwise discarded unless trace logging is enabled. Fabricated + // empties that no override rule filled are the designed + // creative-opportunity path (see `CreativeOpportunitySlot::to_ad_slot`), + // so they log at debug and do not bury the warn. + let mut dropped_slot_supplied: Vec = Vec::new(); + let mut dropped_fabricated: Vec = Vec::new(); bidder.retain(|name, params| { - let is_empty_object = params.as_object().is_some_and(serde_json::Map::is_empty); - !is_empty_object || explicit_bidders.contains(name) + if is_unusable_bidder_params(params) { + if slot_supplied.contains(name) { + dropped_slot_supplied.push(name.clone()); + } else { + dropped_fabricated.push(name.clone()); + } + return false; + } + true }); + if !dropped_slot_supplied.is_empty() { + // Sorted so the line is stable across `HashMap` iteration order. + dropped_slot_supplied.sort(); + log::warn!( + "prebid: dropping slot '{}' bidders [{}] — slot supplied empty or non-object params; slot falls back to its stored request", + slot.id, + dropped_slot_supplied.join(", ") + ); + } + if !dropped_fabricated.is_empty() { + dropped_fabricated.sort(); + log::debug!( + "prebid: dropping slot '{}' bidders [{}] — expansion fabricated empty params and no override rule filled them; slot falls back to its stored request", + slot.id, + dropped_fabricated.join(", ") + ); + } - // When no eligible PBS bidder params remain (e.g. creative-opportunity - // slots whose PBS params live in stored requests, or a slot whose - // configured bidders all resolved to fabricated empties), tell PBS to - // resolve bidder config from the stored request keyed by this slot ID. + // When no eligible PBS bidder params remain, tell PBS to resolve + // bidder config from the stored request keyed by this slot ID. This + // covers creative-opportunity slots whose PBS params live in stored + // requests, and slots whose configured bidders all resolved to + // unusable params and were dropped above. // - // This cannot fire for a client /auction slot that carries real - // inline params: the JS adapter injects a `trustedServer` entry, and - // any bidder with params survives the drop above. It falls back only - // when nothing eligible remains — the same no-bid outcome as before. + // Note the fallback set is wider than "slots with no inline params": + // a slot carrying real inline params for a bidder absent from + // `config.bidders` also lands here — that bidder is never expanded, + // the configured bidders fabricate empties, the drop clears them, and + // the real params are lost. That param loss is pre-existing (see the + // debug log in the build loop); it is called out here because it + // means an operator with `config.bidders` set but no stored request + // provisioned for such a slot gets a no-bid. let storedrequest = if bidder.is_empty() { Some(ImpStoredRequest { id: slot.id.clone(), @@ -6627,10 +6700,58 @@ bidders = ["kargo", "triplelift", "criteo"] } #[test] - fn to_openrtb_preserves_an_explicitly_empty_bidder() { - // A publisher-supplied empty `{}` is a real (if misconfigured) signal and - // must survive so the misconfiguration stays visible — unlike a fabricated - // empty, which is dropped. + fn to_openrtb_prefers_direct_bidder_params_over_fabricated_empty() { + // A slot can carry BOTH a direct bidder entry (valid inline params) and a + // `trustedServer` entry whose `bidderParams` omits that bidder — the + // expansion fabricates an empty `{}` for the omitted bidder. Direct real + // params must win over that fabricated empty, and the trustedServer-supplied + // params for the other bidder must still ship. + // + // Looped because `slot.bidders` HashMap iteration order is randomized per + // map; the merge must be order-independent every time. + let config = parse_prebid_toml( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +bidders = ["kargo", "triplelift"] +"#, + ); + + for _ in 0..64 { + let slot = make_slot( + "ad-header-0", + HashMap::from([ + ("kargo".to_string(), json!({ "placementId": "direct-1" })), + ( + TRUSTED_SERVER_BIDDER.to_string(), + json!({ BIDDER_PARAMS_KEY: { "triplelift": { "inventoryCode": "tl-1" } } }), + ), + ]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config.clone(), &request); + let params = bidder_params(&ortb); + + assert_eq!( + params["kargo"]["placementId"], "direct-1", + "direct bidder params must win over a fabricated empty from trustedServer expansion" + ); + assert_eq!( + params["triplelift"]["inventoryCode"], "tl-1", + "trustedServer-supplied bidder params must still ship" + ); + } + } + + #[test] + fn to_openrtb_drops_an_explicitly_empty_bidder() { + // PBS cannot distinguish a publisher-supplied empty `{}` from a fabricated + // one — they are identical bytes on the wire and PBS rejects both. So an + // explicit empty is dropped just like a fabricated one, rather than shipped + // as `"bidder": {"kargo": {}}` (which would fail the whole auction). With no + // eligible bidder left, the slot falls back to its stored request. let config = parse_prebid_toml( r#" [integrations.prebid] @@ -6644,12 +6765,126 @@ bidders = ["kargo"] let request = make_auction_request(vec![slot]); let ortb = call_to_openrtb(config, &request); - let params = bidder_params(&ortb); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + assert!( + prebid.get("bidder").is_none(), + "should drop an explicitly supplied empty bidder object, not ship it" + ); assert_eq!( - params["kargo"], - json!({}), - "should preserve an explicitly supplied empty bidder object" + prebid["storedrequest"]["id"], "ad-header-0", + "should fall back to the stored request once the empty bidder is dropped" + ); + } + + #[test] + fn to_openrtb_direct_empty_does_not_clobber_trusted_server_params() { + // A slot can carry BOTH a direct empty entry and a `trustedServer` entry + // whose `bidderParams` supplies real params for the same bidder. The direct + // empty must NOT overwrite the real params — otherwise the slot ships + // `"bidder": {"kargo": {}}`, the exact invalid payload this path removes. + // Looped because `slot.bidders` HashMap iteration order is randomized. + let config = parse_prebid_toml( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +bidders = ["kargo"] +"#, + ); + + for _ in 0..64 { + let slot = make_slot( + "ad-header-0", + HashMap::from([ + ("kargo".to_string(), json!({})), + ( + TRUSTED_SERVER_BIDDER.to_string(), + json!({ BIDDER_PARAMS_KEY: { "kargo": { "placementId": "ts-1" } } }), + ), + ]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config.clone(), &request); + let params = bidder_params(&ortb); + + assert_eq!( + params["kargo"]["placementId"], "ts-1", + "a direct empty must not clobber real trustedServer-supplied params" + ); + } + } + + #[test] + fn to_openrtb_drops_non_object_bidder_params() { + // Non-object params (e.g. `null`, reachable via a POST that omits `params` + // because `BidConfig.params` is `#[serde(default)]`) are as invalid to PBS + // as `{}` and must be dropped, not shipped as `"bidder": {"kargo": null}`. + let config = parse_prebid_toml( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +bidders = ["kargo"] +"#, + ); + + let slot = make_slot( + "ad-header-0", + HashMap::from([("kargo".to_string(), Json::Null)]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "should drop a null (non-object) bidder params value" + ); + assert_eq!( + prebid["storedrequest"]["id"], "ad-header-0", + "should fall back to the stored request once the null bidder is dropped" + ); + } + + #[test] + fn to_openrtb_falls_back_to_stored_request_for_real_params_of_unconfigured_bidder() { + // Real inline params naming a bidder absent from `config.bidders` do NOT + // ship: the bidder is never expanded, the configured bidders fabricate + // empties, and the drop clears them — so the slot falls back to its stored + // request and the real params are lost. Pins the corrected comment that this + // fallback can fire even when a slot carries real inline params. + let config = parse_prebid_toml( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +bidders = ["appnexus"] +"#, + ); + + let slot = make_ts_slot( + "ad-header-0", + &json!({ "kargo": { "placementId": "kn1" } }), + None, + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "real params for an unconfigured bidder are not shipped" + ); + assert_eq!( + prebid["storedrequest"]["id"], "ad-header-0", + "slot falls back to its stored request when the only inline params name an unconfigured bidder" ); } diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 70a4d6cfd..edc9a3c0a 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -70,6 +70,7 @@ pub mod streaming_processor; pub mod streaming_replacer; pub mod test_support; pub mod tester_cookie; +pub mod trace_cookie; pub mod tsjs; #[cfg(test)] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 578523644..9bdfd0ebd 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -18,6 +18,7 @@ //! into any [`Write`] (a `Vec` for buffered routes, a streaming writer for //! the streaming route). It is not a content-rewriting concern. +use std::borrow::Cow; use std::io::Write; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -28,7 +29,7 @@ use brotli::enc::writer::CompressorWriter; use cookie::CookieJar; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; -use flate2::read::{GzDecoder, ZlibDecoder}; +use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; use futures::StreamExt as _; use http::{HeaderValue, Method, Request, Response, StatusCode, Uri, header}; @@ -60,8 +61,8 @@ use crate::price_bucket::{PriceGranularity, price_bucket}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{ - BodyStreamDecoder, BodyStreamEncoder, Compression, PipelineConfig, STREAM_CHUNK_SIZE, - StreamProcessor, StreamingPipeline, + BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, PipelineConfig, + STREAM_CHUNK_SIZE, StreamProcessor, StreamingPipeline, }; use crate::streaming_replacer::create_url_replacer; @@ -433,6 +434,10 @@ fn process_response_streaming( output_compression: compression, chunk_size: 8192, }; + // Bound the gzip decode path to the same ceiling the buffered writer + // enforces, so a gzip bomb is rejected mid-decode instead of materializing + // its full expansion before the downstream writer can reject it. + let max_decoded_bytes = params.settings.publisher.max_buffered_body_bytes; if is_html { let processor = create_html_stream_processor( @@ -444,7 +449,9 @@ fn process_response_streaming( params.ad_slots_script.map(str::to_string), params.ad_bids_state.clone(), )?; - StreamingPipeline::new(config, processor).process(body_as_reader(body)?, output)?; + StreamingPipeline::new(config, processor) + .with_max_decoded_bytes(max_decoded_bytes) + .process(body_as_reader(body)?, output)?; } else if is_rsc_flight { // RSC Flight responses are length-prefixed (T rows). A naive string replacement will // corrupt the stream by changing byte lengths without updating the prefixes. @@ -454,7 +461,9 @@ fn process_response_streaming( params.request_host, params.request_scheme, ); - StreamingPipeline::new(config, processor).process(body_as_reader(body)?, output)?; + StreamingPipeline::new(config, processor) + .with_max_decoded_bytes(max_decoded_bytes) + .process(body_as_reader(body)?, output)?; } else { let replacer = create_url_replacer( params.origin_host, @@ -462,7 +471,9 @@ fn process_response_streaming( params.request_host, params.request_scheme, ); - StreamingPipeline::new(config, replacer).process(body_as_reader(body)?, output)?; + StreamingPipeline::new(config, replacer) + .with_max_decoded_bytes(max_decoded_bytes) + .process(body_as_reader(body)?, output)?; } Ok(()) @@ -726,10 +737,11 @@ async fn hold_step_decoded_chunk( collect_refs: &AuctionHoldCollectRefs<'_>, ) -> Result> { let mut ready = Vec::new(); - let bytes = match state.hold.as_mut() { - // Once the hold has been released the chunk streams straight through. - None => chunk.to_vec(), - Some(hold_buffer) => hold_buffer.push(chunk), + let bytes: Cow<'_, [u8]> = match state.hold.as_mut() { + // Once the hold has been released the chunk streams straight through, + // borrowed rather than copied. + None => Cow::Borrowed(chunk), + Some(hold_buffer) => Cow::Owned(hold_buffer.push(chunk)), }; match process_and_encode_chunk(processor, encoder, &bytes, false, "Failed to process chunk") { Ok(Some(encoded)) => ready.push(encoded), @@ -764,17 +776,15 @@ async fn hold_collect_close_tail( .dispatched .take() .expect("should have dispatched auction to collect"); - collect_stream_auction( - dispatched, - state.telemetry.take(), - collect_refs.price_granularity, - collect_refs.ad_bids_state, - collect_refs.orchestrator, - collect_refs.services, - collect_refs.settings, - collect_refs.request_origin, - ) - .await; + let deps = AuctionCollectDeps { + price_granularity: collect_refs.price_granularity, + ad_bids_state: collect_refs.ad_bids_state, + orchestrator: collect_refs.orchestrator, + services: collect_refs.services, + settings: collect_refs.settings, + request_origin: collect_refs.request_origin.to_string(), + }; + collect_stream_auction(dispatched, state.telemetry.take(), &deps).await; // Collection reached a terminal result; disarm only now so a drop while the // collect await above was still pending is reported. state.dispatched.disarm(); @@ -1588,12 +1598,14 @@ pub async fn stream_publisher_body_async( AuctionCollectCtx { dispatched, telemetry, - price_granularity: params.price_granularity, - ad_bids_state: ¶ms.ad_bids_state, - orchestrator, - services, - settings, - request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + deps: AuctionCollectDeps { + price_granularity: params.price_granularity, + ad_bids_state: ¶ms.ad_bids_state, + orchestrator, + services, + settings, + request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + }, }, ) .await @@ -1934,10 +1946,19 @@ impl AuctionTelemetryCarry { } } -/// Bundles the auction-collection dependencies passed through the streaming helpers. +/// Bundles the auction-collection state passed through the streaming helpers. struct AuctionCollectCtx<'a> { dispatched: DispatchedAuction, telemetry: AuctionTelemetryCarry, + deps: AuctionCollectDeps<'a>, +} + +/// Borrowed dependencies of the auction collect step. +/// +/// Split from the per-auction state above because `dispatched` and `telemetry` +/// are moved out at collect time while these stay live for the rest of the +/// streaming loop. +struct AuctionCollectDeps<'a> { price_granularity: PriceGranularity, ad_bids_state: &'a Arc>>, orchestrator: &'a AuctionOrchestrator, @@ -1967,7 +1988,7 @@ async fn stream_html_with_auction_hold( ctx: AuctionCollectCtx<'_>, ) -> Result<(), Report> { if body.is_stream() { - let max_body_bytes = ctx.settings.publisher.max_buffered_body_bytes; + let max_body_bytes = ctx.deps.settings.publisher.max_buffered_body_bytes; return body_close_hold_loop_stream( body, output, @@ -1979,11 +2000,18 @@ async fn stream_html_with_auction_hold( .await; } + // Bound the gzip decode budget to the same ceiling the buffered writer + // enforces, matching the streaming arm above and the no-hold buffered path. + let max_body_bytes = ctx.deps.settings.publisher.max_buffered_body_bytes; let body = body_as_reader(body)?; match compression { Compression::None => body_close_hold_loop(body, output, processor, ctx).await, Compression::Gzip => { - let decoder = GzDecoder::new(body); + // `GzipDecodeReader` decodes concatenated gzip members (RFC 1952) + // and bounds decoded output, unlike `flate2::read::GzDecoder`, which + // silently drops every member after the first — dropping trailing + // markup (potentially including ``) on buffered adapters. + let decoder = GzipDecodeReader::new(body, max_body_bytes); let mut encoder = GzEncoder::new(&mut *output, flate2::Compression::default()); body_close_hold_loop(decoder, &mut encoder, processor, ctx).await?; encoder.finish().change_context(TrustedServerError::Proxy { @@ -2038,12 +2066,15 @@ async fn body_close_hold_loop_stream( let AuctionCollectCtx { dispatched, telemetry, - price_granularity, - ad_bids_state, - orchestrator, - services, - settings, - request_origin, + deps: + AuctionCollectDeps { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + request_origin, + }, } = ctx; let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(compression); @@ -2168,12 +2199,7 @@ async fn body_close_hold_loop( let AuctionCollectCtx { dispatched, mut telemetry, - price_granularity, - ad_bids_state, - orchestrator, - services, - settings, - request_origin, + deps, } = ctx; let mut buffer = vec![0u8; STREAM_CHUNK_SIZE]; let mut hold = Some(BodyCloseHoldBuffer::new()); @@ -2186,17 +2212,7 @@ async fn body_close_hold_loop( let dispatched = dispatched .take() .expect("should have dispatched auction to collect"); - collect_stream_auction( - dispatched, - telemetry.take(), - price_granularity, - ad_bids_state, - orchestrator, - services, - settings, - &request_origin, - ) - .await; + collect_stream_auction(dispatched, telemetry.take(), &deps).await; let held = hold.finish(); write_processed_chunk( @@ -2236,7 +2252,7 @@ async fn body_close_hold_loop( ) { if let Some(dispatched) = dispatched.take() { emit_abandoned_auction( - services, + deps.services, telemetry.observation.take(), dispatched, "stream_process_error", @@ -2250,17 +2266,7 @@ async fn body_close_hold_loop( let dispatched = dispatched .take() .expect("should have dispatched auction to collect"); - collect_stream_auction( - dispatched, - telemetry.take(), - price_granularity, - ad_bids_state, - orchestrator, - services, - settings, - &request_origin, - ) - .await; + collect_stream_auction(dispatched, telemetry.take(), &deps).await; let held = hold .take() @@ -2289,7 +2295,7 @@ async fn body_close_hold_loop( Err(e) => { if let Some(dispatched) = dispatched.take() { emit_abandoned_auction( - services, + deps.services, telemetry.observation.take(), dispatched, "stream_read_error", @@ -2381,20 +2387,22 @@ async fn collect_non_html_auction( ); } -// Private orchestration helper called only from `body_close_hold_loop`, whose -// arguments mirror the fields of `AuctionCollectCtx` it destructures; a separate -// parameter struct would just duplicate that context. -#[allow(clippy::too_many_arguments)] +// Private orchestration helper called only from `body_close_hold_loop`. +// `dispatched` and `telemetry` are moved per collect, so they stay by value +// while the rest of the context is borrowed. async fn collect_stream_auction( dispatched: DispatchedAuction, telemetry: AuctionTelemetryCarry, - price_granularity: PriceGranularity, - ad_bids_state: &Arc>>, - orchestrator: &AuctionOrchestrator, - services: &RuntimeServices, - settings: &Settings, - request_origin: &str, + deps: &AuctionCollectDeps<'_>, ) { + let AuctionCollectDeps { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + request_origin, + } = deps; log::info!("body_close_hold_loop: collecting dispatched auction before held body tail"); let placeholder = mediator_placeholder_request(); let collect_ctx = make_collect_context(settings, services, &placeholder); @@ -2421,7 +2429,7 @@ async fn collect_stream_auction( ); write_bids_to_state( &result.winning_bids, - price_granularity, + *price_granularity, ad_bids_state, settings, request_origin, @@ -3140,9 +3148,9 @@ fn html_escape_for_script(s: &str) -> String { /// /// Every entry also carries the trace fields `hb_auction_id` (when /// `auction_id` is known), `hb_crid` (when the bidder returned a creative ID), -/// and `hb_adm_hash` (when the bid has creative markup) so the client can -/// stamp rendered creatives with a tuple that joins back to the server-side -/// `auction winner:` log lines. +/// `hb_bid_id` (the bid's own `OpenRTB` `id`), and `hb_adm_hash` (when the bid +/// has creative markup) so the client can stamp rendered creatives with a tuple +/// that joins back to the server-side `auction winner:` log lines. pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, granularity: crate::price_bucket::PriceGranularity, @@ -3184,12 +3192,17 @@ pub(crate) fn build_bid_map( serde_json::Value::String(creative_id.clone()), ); } - if let Some(adm_hash) = bid.creative_trace_hash() { + // The bid's own OpenRTB `id`, carried separately from `hb_adid` + // so the client can stamp the exact bid this render came from. + if let Some(ref bid_id) = bid.bid_id { obj.insert( - "hb_adm_hash".to_string(), - serde_json::Value::String(adm_hash), + "hb_bid_id".to_string(), + serde_json::Value::String(bid_id.clone()), ); } + if let Some(hash) = bid.creative_trace_hash() { + obj.insert("hb_adm_hash".to_string(), serde_json::Value::String(hash)); + } // Winning creative dimensions — the bridge sizes the inline // render from these, falling back to the first configured slot // format only when absent, which mis-sizes a multi-size slot. @@ -3202,7 +3215,8 @@ pub(crate) fn build_bid_map( obj.insert("h".to_string(), serde_json::Value::from(bid.height)); } // PBS Cache remains highest priority. APS uses the selected bid ID - // carried by its typed renderer; other providers retain the ad-ID fallback. + // carried by its typed renderer, then other providers use their + // ad ID. The OpenRTB bid ID is the final stable fallback. let renderer_bid_id = bid .renderer .as_ref() @@ -3211,7 +3225,8 @@ pub(crate) fn build_bid_map( .cache_id .as_deref() .or(renderer_bid_id) - .or(bid.ad_id.as_deref()); + .or(bid.ad_id.as_deref()) + .or(bid.bid_id.as_deref()); if let Some(id) = hb_adid { obj.insert( "hb_adid".to_string(), @@ -3234,11 +3249,19 @@ pub(crate) fn build_bid_map( serde_json::Value::String(path.clone()), ); } + // Win/billing notification URLs, fired verbatim by the bridge. + // Per OpenRTB these are the canonical carriers of + // `${AUCTION_PRICE}`, so expand it from the same winning CPM used + // for the creative below — an unexpanded macro would report an + // unresolved clearing price to the SSP, and some reject such + // notifications outright. if let Some(ref nurl) = bid.nurl { - obj.insert("nurl".to_string(), serde_json::Value::String(nurl.clone())); + let nurl = crate::creative::expand_auction_price_macro(nurl, cpm); + obj.insert("nurl".to_string(), serde_json::Value::String(nurl)); } if let Some(ref burl) = bid.burl { - obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); + let burl = crate::creative::expand_auction_price_macro(burl, cpm); + obj.insert("burl".to_string(), serde_json::Value::String(burl)); } if let Some(ref renderer) = bid.renderer { obj.insert( @@ -3279,7 +3302,12 @@ pub(crate) fn build_bid_map( obj.insert("adm".to_string(), serde_json::Value::String(adm)); } } - // Verbose per-bid debug blob is included only under the testing flag. + // Verbose per-bid debug blob only under the testing flag; also + // doubles as the client-side gate for the direct GAM-replace path. + // Deliberately mirrors the bidder-supplied `creative`/`nurl`/`burl` + // verbatim, macros unexpanded: this blob is diagnostic — nothing + // renders or fires from it — and showing what the bidder actually + // sent is the point. if include_debug_bid { obj.insert( "debug_bid".to_string(), @@ -4578,12 +4606,14 @@ mod tests { observation: None, auction_request: None, }, - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), + deps: AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }, }; let mut output = Vec::new(); @@ -5994,6 +6024,72 @@ mod tests { }); } + #[test] + fn stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let state = Arc::new(Mutex::new(None)); + let mut params = OwnedProcessResponseParams { + content_encoding: "gzip".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: Some( + r#""# + .to_string(), + ), + ad_bids_state: state, + auction_observation: None, + auction_request: Some(test_auction_request()), + dispatched_auction: Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 10, + )), + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + // The `` that triggers bid injection lives in the SECOND gzip + // member. `flate2::read::GzDecoder` decodes only the first member, so + // this buffered `Body::Once` body (the non-stream auction arm) proves + // the multi-member decoder now reads every member. + let mut compressed = gzip_encode(b"hello"); + compressed.extend(gzip_encode(b"")); + let body = EdgeBody::from(compressed); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("buffered multi-member gzip auction body should process"); + + let html = String::from_utf8(gzip_decode(&output)).expect("should be valid UTF-8"); + assert!( + html.contains("hello"), + "should decode the first gzip member. Got: {html}" + ); + assert!( + html.contains(""), + "should decode the second gzip member that a single-member decoder drops. Got: {html}" + ); + assert!( + html.contains(".bids=JSON.parse"), + "should inject bids before the carried in the second member. Got: {html}" + ); + }); + } + #[test] fn stream_publisher_body_async_processes_non_html_stream_after_auction_collect() { futures::executor::block_on(async { @@ -6123,6 +6219,169 @@ mod tests { ); } + /// An origin body stream that yields `chunk` once and then stays `Pending` + /// forever without ever signalling EOF — modelling an origin that has sent + /// the document head but not yet finished the response. + fn origin_chunk_then_pending(chunk: bytes::Bytes) -> EdgeBody { + let mut sent = false; + EdgeBody::from_stream(futures::stream::poll_fn(move |_cx| { + if sent { + return std::task::Poll::Pending; + } + sent = true; + std::task::Poll::Ready(Some(Ok::<_, io::Error>(chunk.clone()))) + })) + } + + /// Poll a lazy `Body::Stream` exactly once and return the first emitted + /// chunk. Panics if the first poll is `Pending` (nothing emitted before the + /// origin would need its next chunk) or the stream ends. + fn first_lazy_body_chunk(body: EdgeBody) -> bytes::Bytes { + let mut stream = body + .into_stream() + .expect("streaming finalize should keep a lazy Body::Stream"); + let waker = futures::task::noop_waker(); + let mut cx = std::task::Context::from_waker(&waker); + let next = futures::StreamExt::next(&mut stream); + let mut next = std::pin::pin!(next); + match std::future::Future::poll(next.as_mut(), &mut cx) { + std::task::Poll::Ready(Some(Ok(bytes))) => bytes, + std::task::Poll::Ready(other) => { + panic!("first poll must emit a chunk, got Ready({other:?})") + } + std::task::Poll::Pending => { + panic!("first poll must emit rewritten content before origin EOF, got Pending") + } + } + } + + fn streaming_finalize_response(params: OwnedProcessResponseParams, body: EdgeBody) -> EdgeBody { + let settings = Arc::new(create_test_settings()); + let registry = Arc::new( + IntegrationRegistry::new(&settings).expect("should create integration registry"), + ); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let services = noop_services(); + let response = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, ¶ms.content_type) + .body(EdgeBody::empty()) + .expect("should build response"); + let publisher_response = PublisherResponse::Stream { + response, + body, + params: Box::new(params), + }; + futures::executor::block_on(publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + registry.as_ref(), + orchestrator, + services, + )) + .expect("should build streaming response") + .into_body() + } + + fn html_stream_params( + content_encoding: &str, + dispatched_auction: Option, + ) -> OwnedProcessResponseParams { + OwnedProcessResponseParams { + content_encoding: content_encoding.to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: Some( + r#""# + .to_string(), + ), + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), + dispatched_auction, + price_granularity: crate::price_bucket::PriceGranularity::default(), + } + } + + #[test] + fn streaming_finalize_emits_compressed_html_before_origin_eof() { + // The FCP regression from #849: the lazy body must emit its first + // rewritten, browser-decodable chunk as soon as the origin delivers one + // block — not buffer the whole (compressed) response until EOF. The + // origin here sends one gzip member and then stays Pending; the first + // poll must still yield gzip bytes that decode to the streamed content. + // The page exceeds the deflate decoder's internal output buffer so the + // first decoded block flushes downstream before the (never-arriving) + // EOF, exactly as a real page does. + let mut page = b"".to_vec(); + for i in 0..4000 { + page.extend(format!("

hello world paragraph {i}

").bytes()); + } + let params = html_stream_params("gzip", None); + let body = streaming_finalize_response( + params, + origin_chunk_then_pending(bytes::Bytes::from(gzip_encode(&page))), + ); + + let first = first_lazy_body_chunk(body); + assert!( + !first.is_empty(), + "first emitted gzip chunk must not be empty" + ); + + // Decode the flushed (not finished) gzip prefix the way a browser would + // decode bytes received so far while the response is still open. + let mut decoder = flate2::write::MultiGzDecoder::new(Vec::new()); + decoder + .write_all(&first) + .expect("first chunk must be valid gzip"); + decoder.flush().expect("should flush gzip decoder"); + let decoded = String::from_utf8(decoder.get_ref().clone()).expect("should be valid UTF-8"); + assert!( + decoded.contains("hello world"), + "first poll must emit decodable streamed content before EOF. Got: {decoded}" + ); + } + + #[test] + fn streaming_finalize_auction_hold_emits_prefix_before_origin_eof() { + // The auction-hold path must stream the document prefix (up to the held + // `` tail) before the origin finishes and before the auction is + // collected — otherwise the hold reintroduces the FCP regression. The + // origin sends the head/body prefix (no ``) then stays Pending. + let page = b"

hello

more streamed content here

"; + let params = html_stream_params( + "", + Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 10, + )), + ); + let body = streaming_finalize_response( + params, + origin_chunk_then_pending(bytes::Bytes::from(&page[..])), + ); + + let first = first_lazy_body_chunk(body); + let html = String::from_utf8(first.to_vec()).expect("should be valid UTF-8"); + assert!( + html.contains("hello"), + "auction-hold path must stream the prefix before EOF. Got: {html}" + ); + assert!( + html.contains(".adSlots=JSON.parse"), + "prefix must carry the injected (rewritten) head before EOF. Got: {html}" + ); + assert!( + !html.contains(".bids=JSON.parse"), + "bids inject only at after collection, which the first poll must not wait for. Got: {html}" + ); + } + // (method, status, expected Content-Length after bodiless normalization). // 204 forbids Content-Length (removed); 205 must advertise a zero-length // body; HEAD and 304 legitimately advertise the GET representation length. @@ -6999,6 +7258,56 @@ mod tests { assert!(script.contains("\\u003C/script\\u003E")); } + #[test] + fn bid_map_includes_trace_fields() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + bid.bid_id = Some("bid-abc123".to_string()); + bid.creative_id = Some("cr-98765".to_string()); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map( + &winning_bids, + PriceGranularity::Dense, + &test_settings(), + "", + false, + Some("ts-req-trace1"), + ); + let obj = map["atf_sidebar_ad"] + .as_object() + .expect("should include traced bid"); + + assert_eq!( + obj.get("hb_auction_id").and_then(|v| v.as_str()), + Some("ts-req-trace1"), + "should carry the auction ID" + ); + assert_eq!( + obj.get("hb_crid").and_then(|v| v.as_str()), + Some("cr-98765"), + "should carry the upstream creative ID" + ); + assert_eq!( + obj.get("hb_bid_id").and_then(|v| v.as_str()), + Some("bid-abc123"), + "should carry the upstream bid ID separately" + ); + assert_eq!( + obj.get("hb_adm_hash").and_then(|v| v.as_str()), + Some(crate::auction::adm_trace_hash("
Creative
").as_str()), + "should hash the raw creative markup" + ); + } + #[test] fn bid_map_omits_zero_creative_dimensions() { // Missing OpenRTB w/h parse to 0. Emitting w:0/h:0 would make the @@ -7366,6 +7675,52 @@ mod tests { ); } + #[test] + fn build_bid_map_expands_auction_price_macro_in_notification_urls() { + // Per OpenRTB the win/billing notices are the primary carriers of + // ${AUCTION_PRICE}, and the bridge fires them verbatim. An unexpanded + // macro would report an unresolved clearing price to the SSP, and + // would disagree with the price already substituted into the adm. + let mut winning_bids = HashMap::new(); + let bid = make_bid( + "atf_sidebar_ad", + 1.50, + "examplessp", + "abc123", + "https://ssp.example.com/win?p=${AUCTION_PRICE}", + "https://ssp.example.com/bill?p=${AUCTION_PRICE}", + ); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map( + &winning_bids, + PriceGranularity::Dense, + &test_settings(), + "", + false, + None, + ); + let obj = map + .get("atf_sidebar_ad") + .and_then(|v| v.as_object()) + .expect("should have bid entry"); + + for field in ["nurl", "burl"] { + let url = obj + .get(field) + .and_then(|v| v.as_str()) + .unwrap_or_else(|| panic!("should include {field}")); + assert!( + !url.to_uppercase().contains("AUCTION_PRICE"), + "no literal or encoded ${{AUCTION_PRICE}} macro should survive in {field}: {url}" + ); + assert!( + url.ends_with("?p=1.5"), + "the exact winning CPM should be substituted into {field}: {url}" + ); + } + } + #[test] fn build_bids_script_escapes_line_separators_in_adm() { // U+2028/U+2029 are valid JSON string content but terminate inline @@ -7485,7 +7840,9 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: None, + // Present alongside cache_id/ad_id to prove cache_id still wins + // — bid_id is the last resort, not a co-equal fallback. + bid_id: Some("should-be-ignored-bid-id".to_string()), ad_id: Some("bid-impression-id".to_string()), creative_id: None, renderer: None, @@ -7511,7 +7868,7 @@ mod tests { assert_eq!( obj.get("hb_adid").and_then(|v| v.as_str()), Some("f47447a0-b759-4f2f-9887-af458b79b570"), - "should use cache_id for hb_adid, not ad_id" + "should use cache_id for hb_adid, not ad_id or bid_id" ); assert_eq!( obj.get("hb_cache_host").and_then(|v| v.as_str()), @@ -7541,7 +7898,9 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: None, + // Present alongside ad_id to prove ad_id still wins — bid_id + // is the last resort, not a co-equal fallback. + bid_id: Some("should-be-ignored-bid-id".to_string()), ad_id: Some("ordinary-ad-id".to_string()), creative_id: None, renderer: None, @@ -7580,7 +7939,56 @@ mod tests { } #[test] - fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { + fn bid_map_falls_back_to_bid_id_when_cache_id_and_ad_id_absent() { + // Real shape for bidders like Kargo: no Prebid Cache UUID, no `adid` + // in the OpenRTB response, but `id` (the bid's own identifier) is + // always present per spec. + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(1.00), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + bid_id: Some("019f7e2a-b45b-70b0-a2d1-b651c430700b".to_string()), + ad_id: None, + creative_id: None, + renderer: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map( + &winning_bids, + PriceGranularity::Dense, + &test_settings(), + "", + false, + None, + ); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("019f7e2a-b45b-70b0-a2d1-b651c430700b"), + "should fall back to bid_id when cache_id and ad_id are both absent" + ); + } + + #[test] + fn bid_map_omits_hb_adid_when_cache_id_ad_id_and_bid_id_all_absent() { let mut winning_bids = HashMap::new(); winning_bids.insert( "atf_sidebar_ad".to_string(), @@ -7620,7 +8028,7 @@ mod tests { .expect("should be object"); assert!( obj.get("hb_adid").is_none(), - "should omit hb_adid when no cache_id and no ad_id" + "should omit hb_adid when no cache_id, ad_id, or bid_id" ); } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 4fe066997..bb99f02aa 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1947,6 +1947,18 @@ pub struct DebugConfig { /// un-sanitized creative for diagnostics, so never enable in production. #[serde(default)] pub inject_adm_for_testing: bool, + + /// Expose `GET /_ts/trace`, which toggles the `ts-trace` cookie and + /// redirects to `/`. + /// + /// The cookie makes the TSJS render-trace overlay draw a floating panel + /// summarising every traced slot (render path, bidder, GAM/injected/visible + /// state) plus a confirmation badge on each genuinely-rendered creative. + /// The overlay only surfaces data already exposed on `window.tsjs`, so + /// enabling this leaks nothing new — it is off by default to avoid shipping + /// a live toggle route on deployments that never asked for it. + #[serde(default)] + pub trace_route_enabled: bool, } /// Tester-cookie endpoint configuration. diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index 51b8d392c..0d1cc7095 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -27,7 +27,7 @@ use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; use brotli::enc::writer::CompressorWriter; use error_stack::{Report, ResultExt as _}; -use flate2::read::{MultiGzDecoder, ZlibDecoder}; +use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; use crate::error::TrustedServerError; @@ -112,6 +112,11 @@ impl Default for PipelineConfig { pub struct StreamingPipeline { config: PipelineConfig, processor: P, + /// Cumulative decoded-byte ceiling for the gzip decode path. Defaults to + /// `usize::MAX` (unbounded); set via [`Self::with_max_decoded_bytes`] on the + /// buffered publisher path so a gzip bomb is rejected mid-decode instead of + /// materializing its full expansion. + max_decoded_bytes: usize, } impl StreamingPipeline

{ @@ -121,7 +126,25 @@ impl StreamingPipeline

{ /// /// No errors are returned by this constructor. pub fn new(config: PipelineConfig, processor: P) -> Self { - Self { config, processor } + Self { + config, + processor, + max_decoded_bytes: usize::MAX, + } + } + + /// Bound the cumulative decoded output of the gzip decode path. + /// + /// Only the gzip reader materializes decoded bytes ahead of the downstream + /// writer; the deflate and brotli read decoders already emit into the + /// caller's fixed-size buffer. Callers that buffer output under a configured + /// ceiling (e.g. `publisher.max_buffered_body_bytes`) should pass that + /// ceiling here so decode rejection cannot be preceded by a large + /// allocation. + #[must_use] + pub fn with_max_decoded_bytes(mut self, max_decoded_bytes: usize) -> Self { + self.max_decoded_bytes = max_decoded_bytes; + self } /// Process a stream from input to output @@ -146,8 +169,10 @@ impl StreamingPipeline

{ (Compression::Gzip, Compression::Gzip) => { // Multi-member decoder: RFC 1952 permits concatenated gzip // members, so a single-member reader would stop after the first. - // Matches the streaming `BodyStreamDecoder` gzip codec. - let decoder = MultiGzDecoder::new(input); + // Shares `GzipStreamDecoder` with the streaming `BodyStreamDecoder` + // gzip codec, so both paths tolerate trailing garbage after the + // final member the same way. + let decoder = GzipDecodeReader::new(input, self.max_decoded_bytes); let mut encoder = GzEncoder::new(output, flate2::Compression::default()); self.process_chunks(decoder, &mut encoder)?; encoder.finish().change_context(TrustedServerError::Proxy { @@ -156,7 +181,7 @@ impl StreamingPipeline

{ Ok(()) } (Compression::Gzip, Compression::None) => { - self.process_chunks(MultiGzDecoder::new(input), output) + self.process_chunks(GzipDecodeReader::new(input, self.max_decoded_bytes), output) } (Compression::Deflate, Compression::Deflate) => { let decoder = ZlibDecoder::new(input); @@ -375,7 +400,8 @@ pub(crate) const STREAM_CHUNK_SIZE: usize = 8192; /// [`flate2::Decompress`] to its [`flate2::Status::StreamEnd`] marker (the /// `write`-based zlib decoder accepts truncated input silently, so the deflate /// arm drives [`flate2::Decompress`] directly). Concatenated gzip members -/// (RFC 1952) are decoded via [`flate2::write::MultiGzDecoder`]. +/// (RFC 1952) are decoded via [`GzipStreamDecoder`], which also tolerates +/// trailing garbage after the final member the way the deflate codec does. pub(crate) struct BodyStreamDecoder { codec: BodyStreamDecoderCodec, /// Cumulative decoded byte count, shared with the codec sinks so the cap is @@ -386,7 +412,7 @@ pub(crate) struct BodyStreamDecoder { enum BodyStreamDecoderCodec { None, - Gzip(flate2::write::MultiGzDecoder), + Gzip(GzipStreamDecoder), Deflate(DeflateStreamDecoder), Brotli(Box>), } @@ -440,6 +466,251 @@ impl Write for BoundedDecodeSink { } } +/// The RFC 1952 gzip member magic number. +const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b]; + +/// Push-style multi-member gzip decoder that tolerates trailing garbage. +/// +/// Decodes concatenated gzip members (RFC 1952) one [`flate2::write::GzDecoder`] +/// at a time. At each member boundary the next bytes are sniffed for the gzip +/// magic number: a match starts the next member, anything else is treated as +/// trailing garbage and silently dropped — matching GNU gzip, the deflate +/// codec's tolerance, and the single-member `read::GzDecoder` the buffered +/// pipeline used before multi-member support. `flate2`'s own `MultiGzDecoder` +/// instead errors on any post-member bytes that do not parse as a new header. +/// +/// Tolerance never weakens integrity checks *inside* a member: a truncated or +/// corrupt member (bad CRC, bad length, incomplete deflate stream) still fails +/// at [`Self::decode`] or [`Self::finish`]. Trailing bytes that happen to start +/// with the magic number are decoded as a member and fail if they are not one. +struct GzipStreamDecoder { + state: GzipStreamState, +} + +enum GzipStreamState { + /// Decoding a gzip member (header, deflate stream, or trailer). + Member(flate2::write::GzDecoder), + /// A member fully decoded and validated; sniffing whether the next bytes + /// start another member. `magic_prefix_seen` records that the first magic + /// byte (`0x1f`) arrived, possibly at the end of an earlier chunk. + Boundary { + sink: BoundedDecodeSink, + magic_prefix_seen: bool, + }, + /// Non-member bytes followed a completed member; all further input is + /// dropped. + TrailingGarbage(BoundedDecodeSink), + /// Transient placeholder while ownership moves between states. Never + /// observable by callers. + Poisoned, +} + +impl GzipStreamDecoder { + fn new(decoded_bytes: Rc>, max_decoded_bytes: usize) -> Self { + Self { + state: GzipStreamState::Member(flate2::write::GzDecoder::new(BoundedDecodeSink::new( + decoded_bytes, + max_decoded_bytes, + ))), + } + } + + /// Decode one compressed chunk, returning the decoded bytes it produced. + /// + /// # Errors + /// + /// Returns an error if a member is corrupt, its trailer fails validation, + /// or the decoded budget is exceeded. + fn decode(&mut self, chunk: &[u8]) -> io::Result> { + let mut input = chunk; + while !input.is_empty() { + match &mut self.state { + GzipStreamState::Member(decoder) => { + let consumed = decoder.write(input)?; + if consumed > 0 { + input = &input[consumed..]; + continue; + } + // A zero-byte write on non-empty input means the member + // (deflate stream plus trailer) is complete: validate the + // trailer and start sniffing for the next member. + let GzipStreamState::Member(decoder) = + std::mem::replace(&mut self.state, GzipStreamState::Poisoned) + else { + unreachable!("state was matched as Member above"); + }; + let sink = decoder.finish()?; + self.state = GzipStreamState::Boundary { + sink, + magic_prefix_seen: false, + }; + } + GzipStreamState::Boundary { + magic_prefix_seen, .. + } => { + let expected = if *magic_prefix_seen { + GZIP_MAGIC[1] + } else { + GZIP_MAGIC[0] + }; + if input[0] != expected { + self.enter_trailing_garbage(); + continue; + } + if !*magic_prefix_seen { + *magic_prefix_seen = true; + input = &input[1..]; + continue; + } + // Full magic number seen: start the next member, replaying + // the two magic bytes consumed during sniffing. + input = &input[1..]; + let GzipStreamState::Boundary { sink, .. } = + std::mem::replace(&mut self.state, GzipStreamState::Poisoned) + else { + unreachable!("state was matched as Boundary above"); + }; + let mut decoder = flate2::write::GzDecoder::new(sink); + decoder.write_all(&GZIP_MAGIC)?; + self.state = GzipStreamState::Member(decoder); + } + GzipStreamState::TrailingGarbage(_) => break, + GzipStreamState::Poisoned => { + // A prior call errored mid-transition, leaving the state + // poisoned. No current caller retries after an error, but + // returning an error (rather than `unreachable!`, which + // aborts the Wasm instance) keeps a re-entrant caller safe. + return Err(io::Error::other( + "gzip decoder previously failed; stream is unusable", + )); + } + } + } + Ok(self.take_decoded()) + } + + /// Finalize the stream at end of input, returning any remaining decoded bytes. + /// + /// # Errors + /// + /// Returns an error if the final member is truncated or its trailer fails + /// validation. Trailing garbage after a completed member is not an error. + fn finish(&mut self) -> io::Result> { + match &mut self.state { + GzipStreamState::Member(decoder) => { + decoder.try_finish()?; + Ok(std::mem::take(&mut decoder.get_mut().buffer)) + } + GzipStreamState::Boundary { + sink, + magic_prefix_seen, + } => { + if *magic_prefix_seen { + log::warn!( + "gzip body ends with a lone trailing byte after the final member; ignoring it" + ); + } + Ok(std::mem::take(&mut sink.buffer)) + } + GzipStreamState::TrailingGarbage(sink) => Ok(std::mem::take(&mut sink.buffer)), + GzipStreamState::Poisoned => { + // See the `Poisoned` arm in `decode`: a prior mid-transition + // error left the decoder unusable; error cleanly rather than + // aborting the Wasm instance via `unreachable!`. + Err(io::Error::other( + "gzip decoder previously failed; stream is unusable", + )) + } + } + } + + /// Drop the rest of the input, keeping the sink so already-decoded bytes + /// can still be drained. + fn enter_trailing_garbage(&mut self) { + log::warn!("gzip body has trailing bytes after the final member; ignoring them"); + let sink = match std::mem::replace(&mut self.state, GzipStreamState::Poisoned) { + GzipStreamState::Boundary { sink, .. } => sink, + _ => unreachable!("trailing garbage is only entered from the boundary state"), + }; + self.state = GzipStreamState::TrailingGarbage(sink); + } + + /// Take the decoded bytes accumulated in the sink so far. + fn take_decoded(&mut self) -> Vec { + match &mut self.state { + GzipStreamState::Member(decoder) => std::mem::take(&mut decoder.get_mut().buffer), + GzipStreamState::Boundary { sink, .. } | GzipStreamState::TrailingGarbage(sink) => { + std::mem::take(&mut sink.buffer) + } + GzipStreamState::Poisoned => { + unreachable!("poisoned state is never left in place across calls") + } + } + } +} + +/// [`Read`] adapter that decodes multi-member gzip through [`GzipStreamDecoder`]. +/// +/// Used by the buffered [`StreamingPipeline`] and the buffered auction hold path +/// so both share the streaming path's trailing-garbage tolerance and multi-member +/// support instead of erroring (or dropping members) like +/// `flate2::read::MultiGzDecoder`/`GzDecoder`. +/// +/// `max_decoded_bytes` bounds cumulative decoded output: each compressed block's +/// expansion is charged against the budget from inside [`GzipStreamDecoder`]'s +/// bounded sink, so a decompression bomb is rejected mid-decode rather than fully +/// materialized before a downstream writer (e.g. `BoundedWriter`) can act. Pass +/// `usize::MAX` where an upstream/downstream stage already bounds the size. +pub(crate) struct GzipDecodeReader { + input: R, + decoder: GzipStreamDecoder, + raw: Vec, + decoded: Vec, + position: usize, + finished: bool, +} + +impl GzipDecodeReader { + pub(crate) fn new(input: R, max_decoded_bytes: usize) -> Self { + Self { + input, + decoder: GzipStreamDecoder::new(Rc::new(Cell::new(0)), max_decoded_bytes), + raw: vec![0_u8; STREAM_CHUNK_SIZE], + decoded: Vec::new(), + position: 0, + finished: false, + } + } +} + +impl Read for GzipDecodeReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + loop { + if self.position < self.decoded.len() { + let available = self.decoded.len() - self.position; + let amount = available.min(buf.len()); + buf[..amount].copy_from_slice(&self.decoded[self.position..self.position + amount]); + self.position += amount; + return Ok(amount); + } + if self.finished { + return Ok(0); + } + let read = self.input.read(&mut self.raw)?; + if read == 0 { + self.decoded = self.decoder.finish()?; + self.finished = true; + } else { + self.decoded = self.decoder.decode(&self.raw[..read])?; + } + self.position = 0; + } + } +} + /// Charge `len` decoded bytes against `decoded_bytes`, erroring if the /// cumulative total would exceed `max_decoded_bytes`. fn charge_decoded( @@ -463,6 +734,28 @@ fn charge_decoded( Ok(()) } +/// Append `data` to `output`, growing capacity with amortized doubling but +/// never past `max_decoded_bytes + 1`. +/// +/// A plain `Vec` doubles on growth, so accumulating a decode up to an N-byte +/// ceiling can leave the buffer with ~2N capacity. Capping the reservation at +/// the decoded limit keeps a decompression bomb from ballooning the accumulator +/// to roughly twice the configured ceiling before the cumulative-byte charge +/// rejects it. +fn extend_capped(output: &mut Vec, data: &[u8], max_decoded_bytes: usize) { + let needed = output.len() + data.len(); + if needed > output.capacity() { + let target = output + .capacity() + .saturating_mul(2) + .max(needed) + .min(max_decoded_bytes.saturating_add(1)) + .max(needed); + output.reserve_exact(target - output.len()); + } + output.extend_from_slice(data); +} + /// Streaming zlib decoder that tracks whether the stream reached its end /// marker, so truncated deflate bodies fail at finalization. struct DeflateStreamDecoder { @@ -487,6 +780,17 @@ impl DeflateStreamDecoder { charge_decoded(&self.decoded_bytes, self.max_decoded_bytes, len) } + /// The number of scratch bytes the inflater may write this step: the + /// remaining budget plus one — so a decompression bomb produces exactly one + /// byte past the ceiling, which [`Self::charge`] then rejects — capped at + /// `scratch_len`. + fn decode_window(&self, scratch_len: usize) -> usize { + let remaining = self + .max_decoded_bytes + .saturating_sub(self.decoded_bytes.get()); + remaining.saturating_add(1).min(scratch_len) + } + /// Decode as much of `chunk` as possible, draining any output the inflater /// can still produce once all input is consumed. /// @@ -497,34 +801,46 @@ impl DeflateStreamDecoder { /// makes no further progress, so those pending bytes are never stranded and /// a valid stream is not mistaken for a truncated one at `finish`. fn decode(&mut self, chunk: &[u8]) -> Result, Report> { - let mut output = Vec::with_capacity(STREAM_CHUNK_SIZE); + let mut output = Vec::new(); let mut offset = 0usize; + // The inflater writes into this fixed-size scratch buffer, never a + // growing `Vec`. `decompress_vec` fills a doubled `Vec` to its full + // capacity before the produced bytes can be charged, so a bomb could + // materialize ~2× the cap before rejection. A fixed scratch window + // bounded by the remaining budget produces at most one byte past the + // ceiling per step, which `charge` rejects before it is appended. + let mut scratch = [0_u8; STREAM_CHUNK_SIZE]; // Trailing bytes after the zlib end marker are ignored, matching the // read-based decoder used by the buffered pipeline. while !self.stream_ended { - if output.len() == output.capacity() { - output.reserve(STREAM_CHUNK_SIZE); - } + let window = self.decode_window(scratch.len()); let before_in = self.decompress.total_in(); let before_out = self.decompress.total_out(); let status = self .decompress - .decompress_vec(&chunk[offset..], &mut output, flate2::FlushDecompress::None) + .decompress( + &chunk[offset..], + &mut scratch[..window], + flate2::FlushDecompress::None, + ) .change_context(TrustedServerError::Proxy { message: "Failed to decode deflate publisher body chunk".to_string(), })?; let consumed = (self.decompress.total_in() - before_in) as usize; let produced = (self.decompress.total_out() - before_out) as usize; offset += consumed; + // Charge before appending so a bomb errors before its bytes are + // buffered. self.charge(produced)?; + extend_capped(&mut output, &scratch[..produced], self.max_decoded_bytes); match status { flate2::Status::StreamEnd => self.stream_ended = true, flate2::Status::Ok | flate2::Status::BufError => { - // Stop only when the inflater is starved for input: it made - // no progress and there is still spare output capacity, so - // the stall is missing input (arriving in a later chunk, or - // resolved at `finish`), not an exhausted output buffer. - if consumed == 0 && produced == 0 && output.len() < output.capacity() { + // The write window is always at least one byte, so no + // progress means the inflater is starved for input (arriving + // in a later chunk, or resolved at `finish`), not an + // exhausted output buffer. + if consumed == 0 && produced == 0 { break; } } @@ -541,19 +857,22 @@ impl DeflateStreamDecoder { /// truncated stream makes no further progress and errors. fn finish(&mut self) -> Result, Report> { let mut output = Vec::new(); + // Same fixed scratch buffer as `decode`: the terminal flush can still + // expand a withheld bomb, so it must be bounded by the remaining budget + // rather than draining into a doubling `Vec`. + let mut scratch = [0_u8; STREAM_CHUNK_SIZE]; while !self.stream_ended { - if output.len() == output.capacity() { - output.reserve(STREAM_CHUNK_SIZE); - } + let window = self.decode_window(scratch.len()); let before_out = self.decompress.total_out(); let status = self .decompress - .decompress_vec(&[], &mut output, flate2::FlushDecompress::Finish) + .decompress(&[], &mut scratch[..window], flate2::FlushDecompress::Finish) .change_context(TrustedServerError::Proxy { message: "Failed to finalize deflate publisher body decoder".to_string(), })?; let produced = (self.decompress.total_out() - before_out) as usize; self.charge(produced)?; + extend_capped(&mut output, &scratch[..produced], self.max_decoded_bytes); match status { flate2::Status::StreamEnd => self.stream_ended = true, flate2::Status::Ok | flate2::Status::BufError => { @@ -578,8 +897,9 @@ impl BodyStreamDecoder { let decoded_bytes = Rc::new(Cell::new(0usize)); let codec = match compression { Compression::None => BodyStreamDecoderCodec::None, - Compression::Gzip => BodyStreamDecoderCodec::Gzip(flate2::write::MultiGzDecoder::new( - BoundedDecodeSink::new(Rc::clone(&decoded_bytes), max_decoded_bytes), + Compression::Gzip => BodyStreamDecoderCodec::Gzip(GzipStreamDecoder::new( + Rc::clone(&decoded_bytes), + max_decoded_bytes, )), Compression::Deflate => BodyStreamDecoderCodec::Deflate(DeflateStreamDecoder::new( Rc::clone(&decoded_bytes), @@ -611,15 +931,13 @@ impl BodyStreamDecoder { Ok(chunk) } BodyStreamDecoderCodec::Gzip(decoder) => { - decoder - .write_all(&chunk) + // The sink charges the decoded bytes as the decoder writes them. + let decoded = decoder + .decode(&chunk) .change_context(TrustedServerError::Proxy { message: "Failed to decode gzip publisher body chunk".to_string(), })?; - // The sink charged the decoded bytes during `write_all`. - Ok(bytes::Bytes::from(std::mem::take( - &mut decoder.get_mut().buffer, - ))) + Ok(bytes::Bytes::from(decoded)) } BodyStreamDecoderCodec::Deflate(decoder) => { Ok(bytes::Bytes::from(decoder.decode(&chunk)?)) @@ -641,12 +959,9 @@ impl BodyStreamDecoder { match &mut self.codec { BodyStreamDecoderCodec::None => Ok(Vec::new()), BodyStreamDecoderCodec::Gzip(decoder) => { - decoder - .try_finish() - .change_context(TrustedServerError::Proxy { - message: "Failed to finalize gzip publisher body decoder".to_string(), - })?; - Ok(std::mem::take(&mut decoder.get_mut().buffer)) + decoder.finish().change_context(TrustedServerError::Proxy { + message: "Failed to finalize gzip publisher body decoder".to_string(), + }) } BodyStreamDecoderCodec::Deflate(decoder) => decoder.finish(), BodyStreamDecoderCodec::Brotli(decoder) => { @@ -983,6 +1298,60 @@ mod tests { ); } + fn zlib_compress(data: &[u8]) -> Vec { + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::best()); + encoder + .write_all(data) + .expect("should write deflate test input"); + encoder.finish().expect("should finish deflate encoding") + } + + #[test] + fn deflate_decode_caps_buffer_capacity_at_decoded_limit() { + // Regression for the ~2× allocation: `decompress_vec` doubled the output + // `Vec` and let the inflater fill the doubled capacity before the cap + // check ran, so a 16 MiB limit peaked at 32 MiB. The fixed-scratch + // decoder must decode a body sitting exactly at the cap without letting + // its accumulator balloon toward twice the ceiling. + let cap = STREAM_CHUNK_SIZE * 16; + let payload = vec![b'a'; cap]; + let compressed = zlib_compress(&payload); + let mut decoder = DeflateStreamDecoder::new(Rc::new(Cell::new(0)), cap); + + let decoded = decoder + .decode(&compressed) + .expect("a body exactly at the cap must decode"); + + assert_eq!(decoded.len(), cap, "should decode the whole payload"); + assert!( + decoded.capacity() <= cap + STREAM_CHUNK_SIZE, + "decode buffer must not balloon toward ~2× the cap: cap={cap} capacity={}", + decoded.capacity() + ); + } + + #[test] + fn deflate_decode_rejects_bomb_before_expanding_past_limit() { + // A tiny compressed input that expands to far more than twice the cap + // must be rejected by the cumulative charge, not decoded in full first. + let cap = STREAM_CHUNK_SIZE * 16; + let bomb = zlib_compress(&vec![0_u8; cap * 64]); + assert!( + bomb.len() < cap, + "test precondition: the bomb's compressed form must stay well under the cap" + ); + let mut decoder = DeflateStreamDecoder::new(Rc::new(Cell::new(0)), cap); + + let err = decoder + .decode(&bomb) + .expect_err("a bomb expanding past the cap must error"); + + assert!( + format!("{err:?}").contains("decoded size exceeded"), + "should report the cumulative decoded cap: {err:?}" + ); + } + fn gzip_member(data: &[u8]) -> Vec { let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); encoder @@ -991,6 +1360,35 @@ mod tests { encoder.finish().expect("should finish gzip encoding") } + #[test] + fn gzip_decode_reader_rejects_bomb_before_materializing_past_limit() { + // A tiny gzip member expanding far past the cap must be rejected by the + // reader's bounded sink mid-decode. Before this bound the reader used a + // `usize::MAX` budget and decoded the whole compressed block into its + // buffer (several MiB) before any downstream writer could reject it. + let cap = STREAM_CHUNK_SIZE; + let bomb = gzip_member(&vec![0_u8; cap * 512]); + assert!( + bomb.len() < cap, + "test precondition: the compressed bomb must stay under the cap" + ); + let mut reader = GzipDecodeReader::new(std::io::Cursor::new(bomb), cap); + + let mut sink = Vec::new(); + let err = std::io::copy(&mut reader, &mut sink) + .expect_err("a gzip bomb expanding past the cap must error"); + + assert!( + err.to_string().contains("decoded size exceeded"), + "should report the cumulative decoded cap: {err}" + ); + assert!( + sink.len() <= cap, + "no more than the cap may be emitted before rejection: {} bytes", + sink.len() + ); + } + #[test] fn body_stream_decoder_decodes_multi_member_gzip_single_chunk() { let mut compressed = gzip_member(b"first member "); @@ -1039,6 +1437,117 @@ mod tests { ); } + #[test] + fn body_stream_decoder_ignores_gzip_trailing_bytes() { + let mut with_trailing = gzip_member(b"gzip payload"); + with_trailing.extend_from_slice(b"junk"); + let mut decoder = BodyStreamDecoder::new(Compression::Gzip, usize::MAX); + + let mut decoded = decoder + .decode_chunk(bytes::Bytes::from(with_trailing)) + .expect("complete gzip member should decode") + .to_vec(); + decoded.extend( + decoder + .finish() + .expect("trailing bytes after the final member should be ignored"), + ); + + assert_eq!( + decoded, b"gzip payload", + "should decode the payload and drop trailing junk" + ); + } + + #[test] + fn body_stream_decoder_ignores_gzip_trailing_bytes_split_across_chunks() { + let mut with_trailing = gzip_member(b"first member "); + with_trailing.extend(gzip_member(b"second member")); + with_trailing.extend_from_slice(b"trailing garbage"); + let mut decoder = BodyStreamDecoder::new(Compression::Gzip, usize::MAX); + + let mut decoded = Vec::new(); + // Small pieces exercise every split point, including inside the + // member boundary sniff and inside the garbage itself. + for piece in with_trailing.chunks(3) { + decoded.extend( + decoder + .decode_chunk(bytes::Bytes::copy_from_slice(piece)) + .expect("gzip members followed by garbage should decode"), + ); + } + decoded.extend( + decoder + .finish() + .expect("trailing bytes after the final member should be ignored"), + ); + + assert_eq!( + decoded, b"first member second member", + "should decode every member and drop the trailing garbage" + ); + } + + #[test] + fn body_stream_decoder_ignores_gzip_lone_trailing_magic_prefix_byte() { + // A single 0x1f after the final member could be the start of another + // member; at end of input it must be treated as garbage, not an error. + let mut with_trailing = gzip_member(b"gzip payload"); + with_trailing.push(0x1f); + let mut decoder = BodyStreamDecoder::new(Compression::Gzip, usize::MAX); + + let decoded = decoder + .decode_chunk(bytes::Bytes::from(with_trailing)) + .expect("complete gzip member should decode"); + decoder + .finish() + .expect("a lone trailing magic prefix byte should be ignored"); + + assert_eq!( + decoded.as_ref(), + b"gzip payload", + "should decode the payload and drop the lone trailing byte" + ); + } + + #[test] + fn body_stream_decoder_rejects_trailing_bytes_resembling_gzip_member() { + // Trailing bytes that start with the gzip magic number are decoded as + // a member, so a corrupt pseudo-member still fails: tolerance is + // best-effort and never accepts data that claims to be a member. + let mut with_trailing = gzip_member(b"gzip payload"); + with_trailing.extend_from_slice(&[0x1f, 0x8b, b'j', b'u', b'n', b'k']); + let mut decoder = BodyStreamDecoder::new(Compression::Gzip, usize::MAX); + + let result = decoder + .decode_chunk(bytes::Bytes::from(with_trailing)) + .and_then(|_| decoder.finish().map(|_| ())); + + let err = result.expect_err("garbage that resembles a gzip member must fail"); + assert!( + format!("{err:?}").contains("gzip"), + "should report a gzip decode failure: {err:?}" + ); + } + + #[test] + fn body_stream_decoder_rejects_truncated_gzip_stream() { + let compressed = gzip_member(b"gzip payload that spans more than one deflate block"); + let truncated = &compressed[..compressed.len() / 2]; + let mut decoder = BodyStreamDecoder::new(Compression::Gzip, usize::MAX); + decoder + .decode_chunk(bytes::Bytes::copy_from_slice(truncated)) + .expect("partial gzip input should decode incrementally"); + + let err = decoder + .finish() + .expect_err("a truncated gzip member must fail at finalization"); + assert!( + format!("{err:?}").contains("finalize gzip"), + "should report the gzip finalization failure: {err:?}" + ); + } + /// Verify that `lol_html` fragments text nodes when input chunks split /// mid-text-node. Script rewriters must be fragment-safe — they accumulate /// text fragments internally until `is_last_in_text_node` is true. @@ -1356,6 +1865,47 @@ mod tests { ); } + #[test] + fn test_gzip_pipeline_ignores_trailing_bytes_after_final_member() { + use flate2::read::GzDecoder; + use std::io::Read as _; + + // Arrange + let mut compressed_input = gzip_member(b"hello world"); + compressed_input.extend_from_slice(b"junk"); + + let replacer = StreamingReplacer::new(vec![Replacement { + find: "hello".to_owned(), + replace_with: "hi".to_owned(), + }]); + + let config = PipelineConfig { + input_compression: Compression::Gzip, + output_compression: Compression::Gzip, + chunk_size: 8192, + }; + + let mut pipeline = StreamingPipeline::new(config, replacer); + let mut output = Vec::new(); + + // Act + pipeline + .process(&*compressed_input, &mut output) + .expect("trailing bytes after the final gzip member should be ignored"); + + // Assert + let mut decompressed = Vec::new(); + GzDecoder::new(&*output) + .read_to_end(&mut decompressed) + .expect("should decompress output"); + + assert_eq!( + String::from_utf8(decompressed).expect("should be valid UTF-8"), + "hi world", + "should process the payload and drop trailing junk" + ); + } + #[test] fn test_gzip_to_none_produces_correct_output() { use flate2::write::GzEncoder; diff --git a/crates/trusted-server-core/src/trace_cookie.rs b/crates/trusted-server-core/src/trace_cookie.rs new file mode 100644 index 000000000..583a96238 --- /dev/null +++ b/crates/trusted-server-core/src/trace_cookie.rs @@ -0,0 +1,218 @@ +//! Render-trace toggle endpoint helpers. +//! +//! `GET /_ts/trace` arms (or with `?enabled=false` disarms) the first-party +//! `ts-trace` cookie and redirects to `/`. While the cookie is present, the +//! TSJS render-trace layer draws a visible badge on every traced creative so +//! an operator can see on the page itself that a creative was delivered by +//! Trusted Server — and via which render path (SSAT/GAM or `/auction`). +//! +//! The route is gated behind `[debug] trace_route_enabled` and returns +//! `404 Not Found` while disabled, mirroring the tester-cookie endpoints. The +//! badge only surfaces data already exposed on `window.tsjs`, so the cookie +//! gates visibility, not access. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt}; +use http::{HeaderValue, Response, StatusCode, header}; + +use crate::constants::COOKIE_TS_TRACE; +use crate::error::TrustedServerError; +use crate::settings::Settings; + +/// How long an armed trace cookie lives, in seconds. +/// +/// One hour: long enough for a debugging session across reloads and SPA +/// navigations, short enough that a forgotten toggle expires on its own. +const TRACE_COOKIE_MAX_AGE_SECS: u32 = 3600; + +/// Formats the trace cookie `Set-Cookie` header value. +/// +/// Deliberately host-only (no `Domain` attribute): a `Domain` scoped to +/// `publisher.cookie_domain` would be rejected by the browser during local +/// development against `127.0.0.1`/`localhost`, and the overlay only needs to +/// work on the exact host being debugged. Also neither `HttpOnly` (the TSJS +/// overlay must read it from `document.cookie`) nor `Secure` (the badge is a +/// debug aid that must work through plain-HTTP local dev proxies, and the +/// cookie carries no data worth protecting). +fn format_trace_cookie() -> String { + format!( + "{}=1; Path=/; SameSite=Lax; Max-Age={}", + COOKIE_TS_TRACE, TRACE_COOKIE_MAX_AGE_SECS, + ) +} + +/// Formats the trace cookie clearing `Set-Cookie` header value. +fn format_clear_trace_cookie() -> String { + format!("{}=; Path=/; SameSite=Lax; Max-Age=0", COOKIE_TS_TRACE) +} + +/// Whether the request's query string asks to disarm the trace cookie. +/// +/// Only an explicit `enabled=false` (or `enabled=0`) disarms; any other query +/// — including none at all — arms it, so `GET /_ts/trace` alone switches the +/// overlay on. +fn query_disables(query: Option<&str>) -> bool { + query.is_some_and(|q| { + q.split('&') + .any(|pair| pair == "enabled=false" || pair == "enabled=0") + }) +} + +/// Handles `GET /_ts/trace`. +/// +/// Returns `404 Not Found` while `[debug] trace_route_enabled` is false. When +/// enabled, sets (or with `?enabled=false` clears) the `ts-trace` cookie +/// scoped to `publisher.cookie_domain` and returns `302 Found` redirecting to +/// `/` — landing back on the homepage confirms the toggle round-trip worked. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::InvalidHeaderValue`] if the configured cookie +/// domain cannot be rendered as an HTTP header value. +pub fn handle_trace_mode( + settings: &Settings, + query: Option<&str>, +) -> Result, Report> { + if !settings.debug.trace_route_enabled { + let mut response = Response::new(EdgeBody::empty()); + *response.status_mut() = StatusCode::NOT_FOUND; + return Ok(response); + } + + let cookie_value = if query_disables(query) { + format_clear_trace_cookie() + } else { + format_trace_cookie() + }; + let set_cookie = HeaderValue::from_str(&cookie_value).change_context( + TrustedServerError::InvalidHeaderValue { + message: "trace cookie contains invalid header value".to_string(), + }, + )?; + + let mut response = Response::new(EdgeBody::empty()); + *response.status_mut() = StatusCode::FOUND; + response + .headers_mut() + .insert(header::LOCATION, HeaderValue::from_static("/")); + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("no-store, private"), + ); + response + .headers_mut() + .insert(header::SET_COOKIE, set_cookie); + Ok(response) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::tests::create_test_settings; + + fn trace_enabled_settings() -> Settings { + let mut settings = create_test_settings(); + settings.debug.trace_route_enabled = true; + settings + } + + #[test] + fn trace_route_arms_cookie_and_redirects_to_root() { + let settings = trace_enabled_settings(); + + let response = handle_trace_mode(&settings, None).expect("should build trace response"); + + assert_eq!( + response.status(), + StatusCode::FOUND, + "enabled trace route should redirect" + ); + assert_eq!( + response + .headers() + .get(header::LOCATION) + .and_then(|v| v.to_str().ok()), + Some("/"), + "should redirect to the site root" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store, private"), + "trace route should not be cacheable" + ); + let set_cookie = response + .headers() + .get(header::SET_COOKIE) + .expect("should set trace cookie") + .to_str() + .expect("should render set-cookie as utf-8"); + assert_eq!( + set_cookie, "ts-trace=1; Path=/; SameSite=Lax; Max-Age=3600", + "trace cookie should be host-only with a bounded lifetime" + ); + } + + #[test] + fn trace_route_clears_cookie_when_disabled_by_query() { + let settings = trace_enabled_settings(); + + let response = handle_trace_mode(&settings, Some("enabled=false")) + .expect("should build trace clear response"); + + assert_eq!( + response.status(), + StatusCode::FOUND, + "clearing should still redirect" + ); + let set_cookie = response + .headers() + .get(header::SET_COOKIE) + .expect("should clear trace cookie") + .to_str() + .expect("should render set-cookie as utf-8"); + assert_eq!( + set_cookie, "ts-trace=; Path=/; SameSite=Lax; Max-Age=0", + "trace cookie clear should expire the cookie" + ); + } + + #[test] + fn trace_route_arms_cookie_for_unrelated_query() { + let settings = trace_enabled_settings(); + + let response = handle_trace_mode(&settings, Some("enabled=true&foo=bar")) + .expect("should build trace response"); + + let set_cookie = response + .headers() + .get(header::SET_COOKIE) + .expect("should set trace cookie") + .to_str() + .expect("should render set-cookie as utf-8"); + assert!( + set_cookie.starts_with("ts-trace=1;"), + "non-disabling query should arm the cookie" + ); + } + + #[test] + fn trace_route_is_disabled_by_default() { + let settings = create_test_settings(); + + let response = + handle_trace_mode(&settings, None).expect("should build disabled trace response"); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "disabled trace route should return not found" + ); + assert!( + response.headers().get(header::SET_COOKIE).is_none(), + "disabled trace route should not set a cookie" + ); + } +} diff --git a/crates/trusted-server-js/lib/src/core/auction.ts b/crates/trusted-server-js/lib/src/core/auction.ts index aeea99992..816b8cf59 100644 --- a/crates/trusted-server-js/lib/src/core/auction.ts +++ b/crates/trusted-server-js/lib/src/core/auction.ts @@ -62,6 +62,13 @@ export interface AuctionBid { adomain: string[]; /** Server-side auction ID (response top-level `id` / `ext.ts.auction_id`). */ auctionId?: string; + /** + * The bid's own OpenRTB `id` — the trace key identifying this exact bid in + * the server-side `auction winner:` log line. Kept distinct from + * [`creativeId`], which is the advertiser's creative (`crid`) and is reused + * across bids and slots. + */ + bidId?: string; /** Trace hash of the delivered adm (`ext.ts.adm_hash`, 16 hex chars of SHA-256). */ admHash?: string; } @@ -164,6 +171,7 @@ export function parseAuctionResponse(body: any): AuctionBid[] { typeof trace?.auction_id === 'string' && trace.auction_id !== '' ? trace.auction_id : responseAuctionId, + bidId: typeof bid?.id === 'string' && bid.id !== '' ? bid.id : undefined, admHash: typeof trace?.adm_hash === 'string' && trace.adm_hash !== '' ? trace.adm_hash : undefined, adomain: Array.isArray(bid?.adomain) diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index 4b7880e73..bbd39827e 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -2,7 +2,7 @@ import { renderApsCreative } from '../integrations/aps/render'; import { buildAdRequest, sendAuction } from './auction'; -import { recordRender, stampCreativeTrace } from './trace'; +import { recordRender, stampCreativeTrace, isEffectivelyVisible } from './trace'; import { collectContext } from './context'; import { log } from './log'; import { getAllUnits, firstSize } from './registry'; @@ -23,6 +23,7 @@ type RenderCreativeInlineOptions = { seat: string; creativeId: string; auctionId?: string; + bidId?: string; admHash?: string; }; @@ -70,6 +71,7 @@ export function requestAds( seat: bid.seat, creativeId: bid.creativeId, auctionId: bid.auctionId, + bidId: bid.bidId, admHash: bid.admHash, }); } @@ -99,12 +101,14 @@ function renderCreativeInline({ seat, creativeId, auctionId, + bidId, admHash, }: RenderCreativeInlineOptions): void { const trace = { slotId, path: 'auction' as const, auctionId, + bidId, bidder: seat, creativeId, admHash, @@ -113,7 +117,7 @@ function renderCreativeInline({ const container = findSlot(slotId) as HTMLElement | null; if (!container) { log.warn('renderCreativeInline: slot not found; skipping render', { slotId, seat, creativeId }); - recordRender({ ...trace, rendered: false }); + recordRender({ ...trace, rendered: false, injected: false, visible: false }); return; } @@ -132,6 +136,8 @@ function renderCreativeInline({ const rejectedRecord = recordRender({ ...trace, rendered: false, + injected: false, + visible: false, elementId: container.id || undefined, }); stampCreativeTrace(container, rejectedRecord); @@ -168,9 +174,13 @@ function renderCreativeInline({ // Trace: registry entry + DOM markers joining this creative back to the // server-side auction (matches the `auction delivered creative:` log line). + // The /auction path writes the srcdoc itself, so this is a confirmed TS + // placement (injected: true). const record = recordRender({ ...trace, rendered: true, + injected: true, + visible: isEffectivelyVisible(container), elementId: container.id || undefined, }); stampCreativeTrace(container, record); diff --git a/crates/trusted-server-js/lib/src/core/trace.ts b/crates/trusted-server-js/lib/src/core/trace.ts index 94edd9fc2..2acc50900 100644 --- a/crates/trusted-server-js/lib/src/core/trace.ts +++ b/crates/trusted-server-js/lib/src/core/trace.ts @@ -1,29 +1,462 @@ -// Render-trace registry and DOM markers: joins a creative rendered on the -// page back to the winning server-side auction bid. Every render writes a -// RenderRecord to window.tsjs.renders (keyed by slot ID), stamps the slot -// element with data-ts-* attributes carrying the same trace tuple, and fires -// a 'tsjs:adRendered' CustomEvent so tests and tooling can await renders. +// Render-trace registry, DOM markers, and a floating debug panel: joins a +// creative rendered on the page back to the winning server-side auction bid. +// Every render writes a RenderRecord to window.tsjs.renders (keyed by slot ID), +// stamps the slot element with data-ts-* attributes carrying the same trace +// tuple, and fires a 'tsjs:adRendered' CustomEvent. When the ts-trace cookie is +// armed (via GET /_ts/trace), a Google-Publisher-Console-style overlay panel +// summarises every traced slot so an operator can confirm on the page itself +// that creatives came through Trusted Server — on both the SSAT/GAM and +// /auction render paths. import { log } from './log'; import type { RenderRecord, TsjsApi } from './types'; /** CustomEvent fired on window after each render-trace record is written. */ export const RENDER_EVENT_NAME = 'tsjs:adRendered'; +/** + * Cookie armed by `GET /_ts/trace` (server-side, `ts-trace=1`). While present, + * the floating trace panel is shown so an operator can see on the page itself + * that creatives were delivered by Trusted Server. + */ +const TRACE_COOKIE_NAME = 'ts-trace'; + +/** DOM id of the floating trace panel (body-level overlay). */ +export const TRACE_PANEL_ID = 'ts-render-trace-panel'; + +/** + * Upper bound on `window.tsjs.renderLog`. A publisher page that refreshes its + * slots on every render can produce hundreds of entries in a session, so the + * history is trimmed from the front rather than growing without limit. + */ +const MAX_RENDER_LOG_ENTRIES = 200; + +/** + * Fallback for [`nextRenderSeq`] when `window.tsjs` is unreachable (no DOM, or + * a throwing property access). Never the primary counter — see below. + */ +let fallbackRenderSeq = 0; + +/** + * Allocate the next value for [`RenderRecord.seq`]. + * + * The counter lives on the shared `window.tsjs` object, not in module scope: + * `build-all.mjs` emits core, GPT and every integration as separate + * self-contained IIFEs, each with its own inlined copy of this module. A + * module-scoped counter would therefore restart at 1 in each bundle and hand + * two different renders the same number — duplicate `#1` panel rows and + * badges across the SSAT and `/auction` paths. + */ +function nextRenderSeq(): number { + try { + const ts = (window.tsjs ??= {} as TsjsApi); + const next = Math.max(ts.renderSeq ?? 0, fallbackRenderSeq) + 1; + ts.renderSeq = next; + fallbackRenderSeq = next; + return next; + } catch { + return ++fallbackRenderSeq; + } +} + +/** CSS class of the per-slot confirmation badge (only on honestly-ok slots). */ +export const TRACE_BADGE_CLASS = 'ts-render-badge'; + +/** + * Whether the visible trace overlay is armed (`ts-trace=1` cookie present — + * set via `GET /_ts/trace`, cleared via `GET /_ts/trace?enabled=false`). + */ +export function traceOverlayEnabled(): boolean { + try { + return new RegExp(`(?:^|;\\s*)${TRACE_COOKIE_NAME}=1(?:;|$)`).test(document.cookie); + } catch { + return false; + } +} + +/** Short-form mechanism suffix — only the bridge mechanisms add information. */ +function mechanismSuffix(record: RenderRecord): string { + return record.servedFrom === 'debug-adm' || record.servedFrom === 'pbs-cache' + ? ` (${record.servedFrom})` + : ''; +} + +/** + * Whether an element is effectively visible: connected, non-zero box, and no + * ancestor hiding it via `display:none`, `visibility:hidden`, or `opacity:0`. + * + * The ancestor walk is what catches a slot the publisher holds at `opacity:0` + * on a wrapper until its own ad code reveals it — the slot's own computed + * opacity is `1`, so only walking up exposes the gate. + */ +export function isEffectivelyVisible(el: Element | null): boolean { + try { + if (!el || !(el instanceof HTMLElement) || !el.isConnected) return false; + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return false; + let node: HTMLElement | null = el; + while (node) { + const cs = getComputedStyle(node); + if ( + cs.display === 'none' || + cs.visibility === 'hidden' || + parseFloat(cs.opacity || '1') === 0 + ) { + return false; + } + node = node.parentElement; + } + return true; + } catch { + return false; + } +} + +/** + * Honest per-slot status for the panel, derived from the separate signals: + * - `empty` — GAM reported the slot empty, or nothing was placed. + * - `hidden` — a creative rendered but the slot is not visible (reveal gate). + * - `gam-only`— GAM rendered something, but TS did not place it (can't confirm + * it is the TS creative — cross-origin). + * - `ok` — TS placed a creative and the slot is visible. + */ +type PanelStatus = 'ok' | 'hidden' | 'gam-only' | 'empty'; + +function panelStatus(record: RenderRecord): PanelStatus { + if (!record.rendered || record.gamEmpty === true) return 'empty'; + if (record.visible === false) return 'hidden'; + // `ok` requires a *confirmed* TS placement. Anything else — TS applied + // targeting only (injected false, creative is GAM's and cross-origin + // unreadable), or a path that never reported placement (undefined) — must not + // be claimed as a TS render. Defaulting to gam-only keeps the panel honest + // even if a future render path forgets to set `injected`. + if (record.injected !== true) return 'gam-only'; + return 'ok'; +} + +const STATUS_STYLE: Record = { + ok: { color: '#3fb950', mark: '✓', label: 'ok' }, + hidden: { color: '#d29922', mark: '⚠', label: 'hidden' }, + 'gam-only': { color: '#58a6ff', mark: '◐', label: 'gam-only' }, + empty: { color: '#f85149', mark: '✗', label: 'empty' }, +}; + +/** + * Attach (or replace) the per-slot confirmation badge on a slot element. + * + * Only called for `ok` slots — a TS creative that actually placed and is + * visible — so the green badge on a physical banner is a truthful "this banner + * is the render in the trace panel" marker, not the overclaiming badge the + * first cut shipped. Hidden / gam-only / empty slots deliberately get none. + * + * `pointer-events: none` keeps the badge from intercepting clicks on the ad. + */ +function attachTraceBadge(el: HTMLElement, record: RenderRecord): void { + const style = STATUS_STYLE[panelStatus(record)]; + + const position = getComputedStyle(el).position; + if (position === 'static' || position === '') { + el.style.position = 'relative'; + } + + const badge = document.createElement('div'); + badge.className = TRACE_BADGE_CLASS; + // Lead with the sequence number: it is what ties this badge to a panel row. + badge.textContent = + `TS ${style.mark} #${record.seq}` + + `${record.bidder ? ` · ${record.bidder}` : ''}` + + `${style.label === 'ok' ? '' : ` · ${style.label}`}`; + badge.title = [ + `render: #${record.seq}`, + `slot: ${record.slotId}`, + `auction: ${record.auctionId ?? '—'}`, + `bidder: ${record.bidder ?? '—'}`, + `bid_id: ${record.bidId ?? '—'}`, + `creative: ${record.creativeId ?? '—'}`, + `adm_hash: ${record.admHash ?? '—'}`, + `served: ${record.servedFrom ?? '—'}`, + ].join('\n'); + const s = badge.style; + s.setProperty('position', 'absolute'); + s.setProperty('top', '4px'); + s.setProperty('left', '4px'); + s.setProperty('z-index', '2147483646'); + s.setProperty('pointer-events', 'none'); + s.setProperty('font', '10px/1.5 ui-monospace, Menlo, Consolas, monospace'); + s.setProperty('padding', '1px 5px'); + s.setProperty('color', '#fff'); + s.setProperty('background', style.color); + s.setProperty('border-radius', '3px'); + el.appendChild(badge); +} + +/** + * Remove this element's own trace badge, if it has one. + * + * Must run on *every* stamp, not only the ones that go on to attach a new + * badge: a slot that re-renders into `empty` or `hidden` gets no replacement + * badge, so without an unconditional removal it would keep displaying the green + * or blue badge from its previous render — contradicting the status the panel + * shows for the same slot. + */ +function removeTraceBadge(el: HTMLElement): void { + el.querySelectorAll(`:scope > .${TRACE_BADGE_CLASS}`).forEach((n) => n.remove()); +} + +/** Truncate a long id for the compact panel row while keeping the tail. */ +function short(value: string | undefined, keep = 10): string { + if (!value) return '?'; + return value.length > keep ? `…${value.slice(-keep)}` : value; +} + +/** + * Create (or return) the floating trace panel appended to `document.body`. + * + * A body-level fixed overlay is used deliberately instead of per-slot badges: + * it survives GAM/APS clearing a slot's `innerHTML`, publisher reveal gates + * that hold a slot wrapper at `opacity: 0`, and cross-origin creative iframes — + * none of which a child-of-slot badge can survive. + */ +function ensureTracePanel(): HTMLElement | null { + if (typeof document === 'undefined' || !document.body) return null; + + const existing = document.getElementById(TRACE_PANEL_ID); + if (existing) return existing; + + const panel = document.createElement('div'); + panel.id = TRACE_PANEL_ID; + const s = panel.style; + s.setProperty('position', 'fixed'); + s.setProperty('bottom', '12px'); + s.setProperty('right', '12px'); + s.setProperty('z-index', '2147483647'); + s.setProperty('max-width', '360px'); + s.setProperty('max-height', '45vh'); + s.setProperty('overflow', 'auto'); + s.setProperty('background', 'rgba(17,17,17,0.94)'); + s.setProperty('color', '#eee'); + s.setProperty('font', '11px/1.5 ui-monospace, Menlo, Consolas, monospace'); + s.setProperty('border', '1px solid #333'); + s.setProperty('border-radius', '6px'); + s.setProperty('box-shadow', '0 4px 16px rgba(0,0,0,0.4)'); + s.setProperty('padding', '0'); + document.body.appendChild(panel); + return panel; +} + +/** + * Whether this record is still the live render for its slot — i.e. the entry + * `window.tsjs.renders` currently holds. Every other row in the log has been + * superseded by a later render of the same slot. + * + * Compares by object identity, not by `seq`: the registry and the history hold + * the same record objects, so identity is exact regardless of how sequence + * numbers were allocated. + */ +function isCurrentRender(record: RenderRecord): boolean { + try { + return window.tsjs?.renders?.[record.slotId] === record; + } catch { + return false; + } +} + +/** GAM/injection state summary for the panel's detail line. */ +function stateSummary(record: RenderRecord): string { + const parts: string[] = []; + // GAM's own fill signal, on every render path that has one. Gating this on + // `ssat` would hide it for `gam-refresh`, where "did GAM fill it this time" + // is the whole question. + if (record.gamEmpty !== undefined) { + parts.push(`gam:${record.gamEmpty ? 'empty' : 'filled'}`); + } + if (record.injected !== undefined) { + parts.push(`inj:${record.injected ? 'y' : 'n'}`); + } + parts.push(`vis:${record.visible === false ? 'n' : record.visible ? 'y' : '?'}`); + return parts.join(' · '); +} + +/** + * Copy a record's full JSON to the clipboard and log it — used by the panel's + * click-to-copy so full (untruncated) auction IDs and hashes are debuggable + * without hovering the title or digging in `window.tsjs.renders`. + */ +function copyRecord(record: RenderRecord): void { + const json = JSON.stringify(record, null, 2); + log.info('trace: render record', record); + try { + void navigator.clipboard?.writeText(json); + } catch { + // Clipboard unavailable (insecure context / permissions) — the console + // log above is the fallback. + } +} + +/** Build one slot row for the panel. */ +function buildPanelRow(record: RenderRecord): HTMLElement { + const status = panelStatus(record); + const style = STATUS_STYLE[status]; + + const row = document.createElement('div'); + const rs = row.style; + rs.setProperty('padding', '6px 10px'); + rs.setProperty('border-top', '1px solid #2a2a2a'); + rs.setProperty('border-left', `3px solid ${style.color}`); + rs.setProperty('cursor', 'pointer'); + // Click a row to copy its full record (untruncated IDs/hash) + log it. + row.addEventListener('click', () => copyRecord(record)); + row.title = [ + `render: #${record.seq}`, + `slot: ${record.slotId}`, + `status: ${style.label}`, + `path: ${record.path}`, + `rendered (gam non-empty): ${record.rendered}`, + `gam_empty: ${record.gamEmpty ?? '—'}`, + `injected (ts placed): ${record.injected ?? '—'}`, + `visible: ${record.visible ?? '—'}`, + `auction: ${record.auctionId ?? '—'}`, + `bidder: ${record.bidder ?? '—'}`, + `creative: ${record.creativeId ?? '—'}`, + `ad_id: ${record.adId ?? '—'}`, + `bid_id: ${record.bidId ?? '—'}`, + `adm_hash: ${record.admHash ?? '—'}`, + `served: ${record.servedFrom ?? '—'}`, + `element: ${record.elementId ?? '—'}`, + `renders: ${record.count}`, + ].join('\n'); + + const line1 = document.createElement('div'); + const clock = new Date(record.at).toLocaleTimeString('en-GB', { hour12: false }); + // `current` marks the row still on screen for its slot — the one whose badge, + // if any, is the badge you are looking at. Older rows are history. + const current = isCurrentRender(record) ? ' ◂ current' : ''; + line1.textContent = `#${record.seq} ${clock} ${style.mark} ${record.slotId} · ${style.label}${current}`; + line1.style.setProperty('font-weight', '600'); + line1.style.setProperty('color', style.color); + + const line2 = document.createElement('div'); + line2.style.setProperty('color', '#bbb'); + // An unattributed render (a GAM refresh TS ran no auction for) carries no + // bidder or hash by design. Say that, rather than rendering `? · ?` as if a + // lookup had failed. + const attribution = + record.bidder || record.admHash + ? `${record.bidder ?? '?'} · ${short(record.admHash)}` + : 'no TS attribution'; + line2.textContent = `${record.path}${mechanismSuffix(record)} · ${attribution}`; + + const line3 = document.createElement('div'); + line3.style.setProperty('color', '#777'); + const auction = record.auctionId ? ` · auction ${short(record.auctionId)}` : ''; + // `×N` is this slot's own render count — distinct from the page-global `#seq` + // on line 1, which is what the on-creative badge shows. + line3.textContent = `${stateSummary(record)}${auction} · ×${record.count}`; + + row.append(line1, line2, line3); + return row; +} + +/** + * Rebuild the floating trace panel from `window.tsjs.renders`. + * + * Reads the whole registry each call so the panel always reflects the current + * state; safe to call on every render event. + */ +export function renderTracePanel(): void { + try { + if (!traceOverlayEnabled()) return; + const panel = ensureTracePanel(); + if (!panel) return; + + const renders = window.tsjs?.renders ?? {}; + const slots = Object.values(renders); + // Count only slots that are honestly OK (TS creative placed and visible), + // not merely "GAM said something rendered" — the whole point of the fix. + const ok = slots.filter((r) => panelStatus(r) === 'ok').length; + // Newest render first: on a page that refreshes its slots this reads as a + // timeline rather than a set of counters. + const history = [...(window.tsjs?.renderLog ?? [])].reverse(); + + panel.replaceChildren(); + + const header = document.createElement('div'); + const hs = header.style; + hs.setProperty('display', 'flex'); + hs.setProperty('justify-content', 'space-between'); + hs.setProperty('align-items', 'center'); + hs.setProperty('gap', '8px'); + hs.setProperty('padding', '6px 10px'); + hs.setProperty('position', 'sticky'); + hs.setProperty('top', '0'); + hs.setProperty('background', '#000'); + hs.setProperty('font-weight', '700'); + + const title = document.createElement('span'); + title.textContent = `TS Render Trace · ${ok}/${slots.length} slots ok · ${history.length} renders`; + + const close = document.createElement('button'); + close.textContent = '×'; + close.setAttribute('aria-label', 'Close trace panel'); + const cs = close.style; + cs.setProperty('background', 'transparent'); + cs.setProperty('color', '#eee'); + cs.setProperty('border', '0'); + cs.setProperty('font-size', '14px'); + cs.setProperty('cursor', 'pointer'); + cs.setProperty('line-height', '1'); + close.addEventListener('click', () => panel.remove()); + + header.append(title, close); + panel.appendChild(header); + + const hint = document.createElement('div'); + hint.style.setProperty('padding', '2px 10px 4px'); + hint.style.setProperty('color', '#777'); + hint.style.setProperty('font-size', '9px'); + hint.textContent = 'newest first · click a row to copy its full record · hover for detail'; + panel.appendChild(hint); + + if (history.length === 0) { + const empty = document.createElement('div'); + empty.style.setProperty('padding', '6px 10px'); + empty.style.setProperty('color', '#bbb'); + empty.textContent = 'No creatives traced yet.'; + panel.appendChild(empty); + return; + } + + for (const record of history) { + panel.appendChild(buildPanelRow(record)); + } + } catch (err) { + log.warn('trace: failed to render panel', err); + } +} + /** * Write a render record into `window.tsjs.renders` and fire the render event. * * Repeated records for the same slot (SPA navigation, GPT refresh) overwrite * the previous entry and increment `count`, so the registry always reflects * the latest render while preserving how many renders the slot has seen. + * When the trace overlay is armed, the floating panel is refreshed here — the + * single choke point every render passes through. */ -export function recordRender(record: Omit): RenderRecord { - const full: RenderRecord = { ...record, count: 1, at: Date.now() }; +export function recordRender(record: Omit): RenderRecord { + const full: RenderRecord = { ...record, count: 1, seq: nextRenderSeq(), at: Date.now() }; try { const ts = (window.tsjs ??= {} as TsjsApi); const renders = (ts.renders ??= {}); const prev = renders[record.slotId]; if (prev) full.count = prev.count + 1; renders[record.slotId] = full; + + // Keep each render as its own history entry, trimmed from the front. + const history = (ts.renderLog ??= []); + history.push(full); + if (history.length > MAX_RENDER_LOG_ENTRIES) { + history.splice(0, history.length - MAX_RENDER_LOG_ENTRIES); + } } catch (err) { log.warn('trace: failed to write render record', { slotId: record.slotId, err }); } @@ -33,9 +466,65 @@ export function recordRender(record: Omit): Render // CustomEvent unavailable — registry entry above is still written. log.debug('trace: failed to dispatch render event', { slotId: record.slotId, err }); } + renderTracePanel(); return full; } +/** + * Fields a later signal about an already-recorded render may contribute. + * Identity (`slotId`) and bookkeeping (`seq`, `count`, `at`) are fixed at + * [`recordRender`] time and are never revised. + */ +export type RenderUpdate = Partial>; + +/** Confirmation flags that a later, weaker signal must never clear. */ +const CONFIRMATION_FIELDS = ['rendered', 'injected'] as const; + +/** + * Merge a later signal into an existing render record, **in place**. + * + * One impression can be observed twice: GAM's `slotRenderEnded` and the Prebid + * Universal Creative bridge both describe the same GAM ad request, and a + * deferred ADM placement resolves an animation frame after the render was first + * recorded. Appending a second [`recordRender`] for those would inflate the + * slot's `count`, the history length, the page-global sequence numbers and the + * panel totals — one impression must stay one row. + * + * So the later signal enriches the record instead: `seq`, `count` and `at` are + * left untouched and no new history entry is appended. Because the registry and + * the history hold the *same* object, mutating it updates both. + * + * Confirmations only ever strengthen. A `false` in `patch` cannot clear a + * `true` already on the record, so the weaker GAM-only signal arriving after + * the bridge's confirmed placement does not erase it. + */ +export function updateRender(record: RenderRecord, patch: RenderUpdate): RenderRecord { + try { + const fields = record as unknown as Record; + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) continue; + if ( + value === false && + fields[key] === true && + (CONFIRMATION_FIELDS as readonly string[]).includes(key) + ) { + continue; + } + fields[key] = value; + } + } catch (err) { + log.warn('trace: failed to update render record', { slotId: record.slotId, err }); + } + try { + window.dispatchEvent(new CustomEvent(RENDER_EVENT_NAME, { detail: record })); + } catch (err) { + // CustomEvent unavailable — the mutated record above still stands. + log.debug('trace: failed to dispatch render update event', { slotId: record.slotId, err }); + } + renderTracePanel(); + return record; +} + /** * Stamp an element with `data-ts-*` attributes carrying the trace tuple, so * a creative in the DOM can be joined to the server-side `auction winner:` / @@ -43,7 +532,9 @@ export function recordRender(record: Omit): Render * * Attributes whose record field is absent are removed, so a re-render of the * same element (SPA navigation, GPT refresh) never leaves stale values from a - * previous auction next to the new ones. + * previous auction next to the new ones. These attributes live on the element + * itself, so they survive a later `innerHTML = ''` that clears the slot's + * children (e.g. the GAM adm interceptor) — unlike a child badge would. */ export function stampCreativeTrace(el: Element, record: RenderRecord): void { const attrs: Array<[string, string | undefined]> = [ @@ -53,9 +544,13 @@ export function stampCreativeTrace(el: Element, record: RenderRecord): void { ['data-ts-auction-id', record.auctionId], ['data-ts-bidder', record.bidder], ['data-ts-ad-id', record.adId], + ['data-ts-bid-id', record.bidId], ['data-ts-creative-id', record.creativeId], ['data-ts-adm-hash', record.admHash], ['data-ts-served-from', record.servedFrom], + ['data-ts-gam-empty', record.gamEmpty === undefined ? undefined : String(record.gamEmpty)], + ['data-ts-injected', record.injected === undefined ? undefined : String(record.injected)], + ['data-ts-visible', record.visible === undefined ? undefined : String(record.visible)], ]; try { for (const [name, value] of attrs) { @@ -65,6 +560,22 @@ export function stampCreativeTrace(el: Element, record: RenderRecord): void { el.removeAttribute(name); } } + // Badge any slot that actually shows something, carrying its honest status + // colour: green ✓ for a confirmed TS render, blue ◐ for `gam-only` (GAM + // rendered, TS cannot confirm it as its own). Slots with nothing on screen + // (`empty`) or nothing visible (`hidden`) stay unbadged — there is no + // creative there to label. Never badge the iframe itself. + // + // Any previous badge is dropped first, unconditionally, so a slot that + // re-renders into `empty` or `hidden` sheds the badge from its last render + // instead of contradicting the panel. + if (el instanceof HTMLElement && el.tagName !== 'IFRAME') { + removeTraceBadge(el); + const status = panelStatus(record); + if (traceOverlayEnabled() && (status === 'ok' || status === 'gam-only')) { + attachTraceBadge(el, record); + } + } } catch (err) { log.warn('trace: failed to stamp element', { slotId: record.slotId, err }); } diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 859095a63..7e05c6fd9 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -87,6 +87,17 @@ export interface AuctionBidData { hb_adid?: string; hb_cache_host?: string; hb_cache_path?: string; + /** + * Upstream OpenRTB `bid.id` — the trace key that identifies this exact bid in + * the server-side `auction winner:` log line. + * + * Distinct from `hb_adid`, which is whatever value GAM's Universal Creative + * must echo back for the render bridge to find the bid (a PBS cache UUID, an + * `adid`, or — only when neither exists — the bid ID). Carried for tracing + * only: unlike the `hb_*` keys in `TS_BID_TARGETING_KEYS` this is never set as + * GAM key-value targeting. + */ + hb_bid_id?: string; /** Server-side auction ID — trace key joining this bid to server logs. */ hb_auction_id?: string; /** Upstream creative ID (OpenRTB `crid`), when the bidder returned one. */ @@ -120,7 +131,7 @@ export interface AuctionBidData { } /** How a creative reached the page for a [`RenderRecord`]. */ -export type RenderServedFrom = 'inline' | 'gam' | 'debug-adm' | 'pbs-cache'; +export type RenderServedFrom = 'inline' | 'gam' | 'debug-adm' | 'pbs-cache' | 'prebid'; /** * One entry in `window.tsjs.renders` — the client-side half of the render @@ -130,8 +141,17 @@ export type RenderServedFrom = 'inline' | 'gam' | 'debug-adm' | 'pbs-cache'; export interface RenderRecord { /** Slot the creative was rendered for. */ slotId: string; - /** Which render path produced this record. */ - path: 'auction' | 'ssat'; + /** + * Which render path produced this record. + * + * `ssat` is claimed only for the render that consumes the server-side + * targeting TS just applied — the server-side auction runs once per + * navigation, so a later GAM refresh of the same slot is NOT an SSAT render + * even though `window.tsjs.bids` still holds that auction's data. + * `gam-refresh` is that later render: GAM re-requested the slot and TS cannot + * attribute the returned creative to any TS auction. + */ + path: 'auction' | 'ssat' | 'gam-refresh'; /** Whether a creative actually rendered (false for empty/rejected). */ rendered: boolean; /** Actual DOM element ID the slot resolved to (div_id may be a prefix). */ @@ -142,14 +162,51 @@ export interface RenderRecord { bidder?: string; /** hb_adid (PBS cache UUID or OpenRTB adid). */ adId?: string; + /** + * Upstream OpenRTB `bid.id`, completing the + * (auctionId, slotId, bidder, bidId, creativeId, admHash) tuple that joins + * this render to exactly one server-side `auction winner:` log line. Never + * overloaded onto [`adId`], which carries a different value whenever the bid + * has a PBS cache UUID or an `adid`. + */ + bidId?: string; /** Upstream creative ID (OpenRTB crid). */ creativeId?: string; /** Trace hash of the creative markup (16 hex chars of SHA-256). */ admHash?: string; /** Mechanism that delivered the creative. */ servedFrom?: RenderServedFrom; + /** + * GAM's own `slotRenderEnded.isEmpty` (SSAT/GAM path only). `true` means GAM + * itself reported the slot empty. Undefined on the `/auction` path, which + * never involves GAM. + */ + gamEmpty?: boolean; + /** + * Whether Trusted Server actually placed the creative markup itself: + * `true` for the `/auction` iframe render and for a synchronous + * `injectAdmIntoSlot` placement; `false` when TS only applied GAM targeting + * (prod GAM path — the creative, if any, is GAM's and lives in a cross-origin + * iframe TS cannot read); `undefined` when placement was deferred/unknown. + * + * This is the honest "is it TS's creative" signal — distinct from `rendered` + * (GAM said something rendered) and `visible` (the slot box is on-screen). + */ + injected?: boolean; + /** + * Whether the slot element was effectively visible at record time — non-zero + * box and no ancestor `display:none` / `visibility:hidden` / `opacity:0`. + * Catches slots that "rendered" but are hidden behind a publisher reveal gate. + */ + visible?: boolean; /** How many renders this slot has seen (SPA navigations, refreshes). */ count: number; + /** + * Page-global render sequence, starting at 1 and shared by the trace panel + * row and the on-creative badge. Unlike `count` (per-slot) this is unique + * across the page, so a badge reading `#12` identifies exactly one row. + */ + seq: number; /** Epoch ms when the record was written. */ at: number; } @@ -190,12 +247,32 @@ export interface TsjsApi { adInit?: () => void; /** Render-trace registry: latest render per slot (see [`RenderRecord`]). */ renders?: Record; + /** + * Append-only history of every render, oldest first, bounded to the most + * recent entries. `renders` collapses to one row per slot (useful for + * "did this slot ever render" checks); this keeps each individual render so + * a refreshing page shows a timeline instead of a climbing counter. + */ + renderLog?: RenderRecord[]; /** GPT slot objects TS defined — used to destroy stale slots on SPA navigation. */ prevGptSlots?: unknown[]; /** Guards one-time-per-page enableSingleRequest/enableServices calls. */ servicesEnabled?: boolean; /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; + /** + * Monotonic render generation, bumped by every `adInit()` and by the start of + * every SPA navigation. Async work captures it and re-checks it on completion + * so a result belonging to a superseded route is discarded rather than + * applied to the current one. + */ + renderGeneration?: number; + /** + * Page-global render counter backing [`RenderRecord.seq`]. Lives here, on the + * object every bundle shares, because each generated IIFE inlines its own + * copy of `core/trace` — a module-scoped counter would restart per bundle. + */ + renderSeq?: number; /** * Win/billing beacons already fired, keyed by `slotId|bidIdentity|kind|url`. * Used by the GPT render bridge so a bid's nurl/burl fire at most once even diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index b7aa9b8ac..9b9450dc8 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,6 +1,17 @@ import { log } from '../../core/log'; -import { recordRender, stampCreativeTrace } from '../../core/trace'; -import type { AuctionSlot, AuctionBidData, RenderServedFrom, TsjsApi } from '../../core/types'; +import { + recordRender, + updateRender, + stampCreativeTrace, + isEffectivelyVisible, +} from '../../core/trace'; +import type { + AuctionSlot, + AuctionBidData, + RenderRecord, + RenderServedFrom, + TsjsApi, +} from '../../core/types'; import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, @@ -43,6 +54,159 @@ const TS_BID_TARGETING_KEYS = [ ] as const; const TS_BASE_TARGETING_KEYS = [...TS_BID_TARGETING_KEYS, TS_INITIAL_TARGETING_KEY] as const; +// ---- Orphaned-slot recovery (client re-render / hydration race) ------------ +// A client framework can replace the ad divs *after* GPT slots were bound to +// them: server-rendered React ids (`…-_R_abc_`) are swapped for client ids +// (`…-_r_1_`) during hydration, leaving GPT holding slots whose element no +// longer exists ("defineSlot was called without a corresponding DIV"). GAM +// still fetches a creative for those slots, but it has nowhere to render, so +// the bid is silently wasted. These bound the recovery re-bind. +/** Quiet period after DOM mutations before checking for orphaned slots. */ +const ORPHAN_RECONCILE_DEBOUNCE_MS = 250; +/** How long after an adInit() to keep watching for a re-render. */ +const ORPHAN_RECONCILE_WINDOW_MS = 5000; +/** + * Maximum re-binds per page load. Each re-bind re-requests the affected slots, + * so this is deliberately small: it recovers a hydration swap without letting a + * continuously-mutating page loop on ad requests. + */ +const MAX_ORPHAN_RECONCILE_ATTEMPTS = 2; + +// ---- Render generation ----------------------------------------------------- +// Every `adInit()` and every SPA navigation opens a new generation. Async work +// (a PBS Cache fetch, a debounced orphan re-bind, a GAM render event) captures +// the generation it started in and re-checks it before acting, so a result that +// belongs to a route the page has already left is discarded instead of being +// applied to the route now on screen. + +function currentGeneration(ts: TsjsApi): number { + return ts.renderGeneration ?? 0; +} + +function bumpGeneration(ts: TsjsApi): number { + const next = currentGeneration(ts) + 1; + ts.renderGeneration = next; + return next; +} + +/** Immutable context for one GPT request TS initiated for a slot. */ +interface PendingGptRender { + generation: number; + slotId: string; + bid: AuctionBidData; + attributed: boolean; +} + +/** + * Per-slot FIFO of requests awaiting `slotRenderEnded`. + * + * Publisher-owned GPT slots are reused across SPA routes, so stamping a single + * generation on the slot object is insufficient: route B overwrites that stamp + * before route A's late event arrives. GPT emits one render event per request in + * request order, so retaining each immutable request context lets the old event + * retire only route A's entry while route B's attribution remains queued. + */ +const pendingGptRenders = new WeakMap(); + +function enqueueGptRender(slot: GoogleTagSlot, pending: PendingGptRender): void { + const queue = pendingGptRenders.get(slot) ?? []; + queue.push(pending); + pendingGptRenders.set(slot, queue); +} + +function takeGptRender(slot: GoogleTagSlot): PendingGptRender | undefined { + const queue = pendingGptRenders.get(slot); + const pending = queue?.shift(); + if (queue?.length === 0) pendingGptRenders.delete(slot); + return pending; +} + +/** + * The render record for the GAM impression currently open on a slot. + * + * One GAM ad request is observed twice: `slotRenderEnded` carries GAM's own + * fill signal, and the Prebid Universal Creative bridge carries the proof that + * TS served the markup. They describe the same impression, so whichever arrives + * second enriches the first one's record rather than appending its own — + * otherwise a single creative counts as two renders, inflating the slot's + * `count`, the history length, the page-global sequence numbers and the panel + * totals. + */ +interface OpenImpression { + record: RenderRecord; + /** Generation the impression was opened in. */ + generation: number; + /** `hb_adid` both signals must agree on to be the same impression. */ + adId?: string; + /** Whether `slotRenderEnded` has reported on this impression. */ + gamSeen: boolean; + /** Whether the Universal Creative bridge has reported on this impression. */ + bridgeSeen: boolean; +} + +const openImpressions = new Map(); + +/** PBS Cache fetches still in flight, so navigation can abort them. */ +const inflightCacheFetches = new Set(); + +/** + * Retire everything armed for the route being left. + * + * Runs at the *start* of a navigation, not after `/__ts/page-bids` returns: a + * route whose markup commits faster than that request can otherwise trip the + * orphan watch or land a cache fetch on the new DOM while `ts.adSlots` and + * `ts.bids` still hold the previous route's auction. + */ +function beginNavigation(ts: TsjsApi): void { + bumpGeneration(ts); + stopOrphanWatch(); + for (const controller of inflightCacheFetches) { + controller.abort(); + } + inflightCacheFetches.clear(); + openImpressions.clear(); +} + +/** + * The impression `side` may still enrich, or `undefined` if it must open a new + * one. + * + * Requires the impression to be from the current route, for the same bid, not + * already reported on by this side, and still the slot's live render — the last + * check is what stops a signal that arrives after a *newer* render of the same + * slot from rewriting an impression the page has moved past. + */ +function enrichableImpression( + ts: TsjsApi, + slotId: string, + adId: string | undefined, + side: 'gam' | 'bridge' +): OpenImpression | undefined { + const open = openImpressions.get(slotId); + if (!open) return undefined; + if (open.generation !== currentGeneration(ts)) return undefined; + if (open.adId !== adId) return undefined; + if (side === 'gam' ? open.gamSeen : open.bridgeSeen) return undefined; + if (ts.renders?.[slotId] !== open.record) return undefined; + return open; +} + +function openImpression( + ts: TsjsApi, + slotId: string, + adId: string | undefined, + record: RenderRecord, + side: 'gam' | 'bridge' +): void { + openImpressions.set(slotId, { + record, + generation: currentGeneration(ts), + adId, + gamSeen: side === 'gam', + bridgeSeen: side === 'bridge', + }); +} + // ------------------------------------------------------------------ // googletag type stubs (minimal surface needed by the shim) // ------------------------------------------------------------------ @@ -322,13 +486,29 @@ export function safeAdmIframeSrc(src: string): string | undefined { * 2. Otherwise replace the slot element's content with a sandboxed srcdoc * iframe (no `allow-same-origin` — see [ADM_IFRAME_SANDBOX]). */ -function injectAdmIntoSlot(divId: string, adm: string): void { +/** + * Returns whether the TS creative was placed **synchronously**. The + * animation-frame retry branch resolves after this returns, so it reports + * `false` (placement deferred/unconfirmed) — the render trace must not claim a + * placement that has not happened yet. + * + * That deferred branch reports its real outcome through `onDeferredPlacement` + * once the animation frame runs. Without it a placement that ultimately + * succeeded would stay recorded as unconfirmed and the panel would report + * `gam-only` forever, even though Trusted Server did place the creative. + */ +function injectAdmIntoSlot( + divId: string, + adm: string, + onDeferredPlacement?: (placed: boolean) => void, + mayPlaceDeferred?: () => boolean +): boolean { try { // divId may be the container div (used by GPT slot) or the inner div. // Resolve it the same way the rest of adInit does (exact then prefix) so // a config div_id prefix with a render-time suffix still finds the element. const slotEl = findSlotElementByDivId(divId); - if (!slotEl) return; + if (!slotEl) return false; // Extract the first iframe src from the adm (e.g. mocktioneer creative // wraps a first-party proxy iframe in a div). Reject non-http(s) schemes. @@ -340,27 +520,48 @@ function injectAdmIntoSlot(divId: string, adm: string): void { // Set the GAM iframe src — works even cross-origin (no document access needed). gamIframe.src = innerSrc; log.debug(`[tsjs-gpt] gam-intercept: set iframe src for '${divId}'`); + return true; } else if (innerSrc) { // GAM iframe not yet in DOM (APS renders async after slotRenderEnded). // Retry on next animation frame so APS has a tick to insert its iframe; // if it still isn't there, replace slot content directly. requestAnimationFrame(() => { - const retryIframe = slotEl!.querySelector('iframe') as HTMLIFrameElement | null; - if (retryIframe) { - retryIframe.src = innerSrc; - log.debug(`[tsjs-gpt] gam-intercept: set iframe src (retry) for '${divId}'`); - } else { - slotEl!.innerHTML = ''; - const f = document.createElement('iframe'); - f.style.cssText = 'border:none'; - f.width = String(slotEl!.offsetWidth || 728); - f.height = String(slotEl!.offsetHeight || 90); - f.setAttribute('sandbox', ADM_IFRAME_SANDBOX); - f.src = innerSrc; - slotEl!.appendChild(f); - log.debug(`[tsjs-gpt] gam-intercept: inserted src iframe for '${divId}'`); + let placed = false; + try { + // Validate before touching the DOM. A newer SPA route may reuse the + // same connected publisher slot element, so suppressing only the + // later trace update would still let the old callback overwrite the + // new route's creative. + if (mayPlaceDeferred && !mayPlaceDeferred()) { + onDeferredPlacement?.(false); + return; + } + const retryIframe = slotEl!.querySelector('iframe') as HTMLIFrameElement | null; + if (retryIframe) { + retryIframe.src = innerSrc; + placed = true; + log.debug(`[tsjs-gpt] gam-intercept: set iframe src (retry) for '${divId}'`); + } else { + slotEl!.innerHTML = ''; + const f = document.createElement('iframe'); + f.style.cssText = 'border:none'; + f.width = String(slotEl!.offsetWidth || 728); + f.height = String(slotEl!.offsetHeight || 90); + f.setAttribute('sandbox', ADM_IFRAME_SANDBOX); + f.setAttribute('data-ts-injected-adm', 'true'); + f.src = innerSrc; + slotEl!.appendChild(f); + placed = true; + log.debug(`[tsjs-gpt] gam-intercept: inserted src iframe for '${divId}'`); + } + } catch (err) { + log.warn('[tsjs-gpt] gam-intercept: deferred placement failed', err); } + onDeferredPlacement?.(placed); }); + // Placement deferred to the animation frame — not confirmed yet. The + // callback above corrects the record once it resolves. + return false; } else { // No extractable safe src — replace slot content with a sandboxed srcdoc iframe. slotEl.innerHTML = ''; @@ -373,9 +574,11 @@ function injectAdmIntoSlot(divId: string, adm: string): void { f.srcdoc = adm; slotEl.appendChild(f); log.debug(`[tsjs-gpt] gam-intercept: replaced slot content for '${divId}'`); + return true; } } catch (err) { log.warn('[tsjs-gpt] gam-intercept: error injecting adm', err); + return false; } } @@ -496,6 +699,88 @@ function installInitialLoadDetector(ts: TsjsApi): void { }); } +// Orphan-watch state. Module-scoped so a re-bind cannot stack observers, and +// so the attempt budget is shared across the page load rather than reset by the +// adInit() the watcher itself triggers. +let orphanObserver: MutationObserver | null = null; +let orphanDebounceTimer: ReturnType | undefined; +let orphanWindowTimer: ReturnType | undefined; +let orphanReconcileAttempts = 0; + +function stopOrphanWatch(): void { + orphanObserver?.disconnect(); + orphanObserver = null; + clearTimeout(orphanDebounceTimer); + clearTimeout(orphanWindowTimer); +} + +/** + * TS-defined GPT slots whose bound element is no longer in the document. + * + * Exported for testing. + */ +export function orphanedTsSlots(ts: TsjsApi): GoogleTagSlot[] { + return ((ts.prevGptSlots ?? []) as GoogleTagSlot[]).filter((slot) => { + const elementId = slot?.getSlotElementId?.(); + return !!elementId && !document.getElementById(elementId); + }); +} + +/** + * Watch for a client re-render that orphans TS's GPT slots and re-bind once it + * happens. + * + * Waiting for the divs to merely *exist* (as the SPA hook does) cannot help + * here: at `adInit()` time the server-rendered divs are present — they are + * later *replaced*. So instead of delaying the initial ad request, this detects + * the swap after the fact and re-runs `adInit()`, which destroys the orphaned + * slots and re-binds against the live DOM (reusing the publisher's own slot for + * that div when they have since defined one). + * + * Bounded by {@link MAX_ORPHAN_RECONCILE_ATTEMPTS} and + * {@link ORPHAN_RECONCILE_WINDOW_MS} so a page whose DOM never settles cannot + * spin on ad requests. + * + * Scoped to the generation it was armed in. SPA navigation is exactly the kind + * of DOM churn this observer reacts to, so without that scope a route change + * whose markup commits faster than `/__ts/page-bids` responds would trip the + * debounce while `ts.adSlots`/`ts.bids` still describe the *previous* route — + * re-binding the finished auction onto the new route's divs, issuing another + * billable GAM request, and burning the recovery budget on an ordinary + * navigation. `beginNavigation()` also disconnects it outright. + */ +function watchForOrphanedSlots(ts: TsjsApi): void { + if (typeof MutationObserver === 'undefined' || typeof document === 'undefined') return; + // Re-arm: a previous window may still be open from an earlier adInit(). + stopOrphanWatch(); + if (orphanReconcileAttempts >= MAX_ORPHAN_RECONCILE_ATTEMPTS) return; + + const generation = currentGeneration(ts); + orphanObserver = new MutationObserver(() => { + clearTimeout(orphanDebounceTimer); + orphanDebounceTimer = setTimeout(() => { + // A navigation (or a newer adInit) has superseded the state this watch + // was armed for — recovering against it would apply the old route. + if (currentGeneration(ts) !== generation) { + stopOrphanWatch(); + return; + } + const orphans = orphanedTsSlots(ts); + if (orphans.length === 0) return; + orphanReconcileAttempts += 1; + log.warn( + `[tsjs-gpt] ${orphans.length} TS slot(s) orphaned by a DOM re-render; re-binding`, + orphans.map((slot) => slot.getSlotElementId?.()) + ); + // Stop before re-running: adInit() re-arms the watch itself. + stopOrphanWatch(); + ts.adInit?.(); + }, ORPHAN_RECONCILE_DEBOUNCE_MS); + }); + orphanObserver.observe(document.documentElement, { childList: true, subtree: true }); + orphanWindowTimer = setTimeout(stopOrphanWatch, ORPHAN_RECONCILE_WINDOW_MS); +} + export function installTsAdInit(): void { const ts = (window.tsjs ??= {} as TsjsApi); installInitialLoadDetector(ts); @@ -509,6 +794,10 @@ export function installTsAdInit(): void { if (!g) return; g.cmd?.push(() => { + // A new ad-init opens a new generation: everything armed by the previous + // one (SSAT attribution, orphan watch, in-flight bridge fetches) belongs + // to state this call is about to replace. + const generation = bumpGeneration(ts); // Destroy previously defined TS slots before redefining for the new page. if (ts.prevGptSlots && ts.prevGptSlots.length > 0) { g.destroySlots?.(ts.prevGptSlots as GoogleTagSlot[]); @@ -592,6 +881,25 @@ export function installTsAdInit(): void { if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); }); gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); + // Arm SSAT attribution for the next render of this slot only. Without + // this, every later publisher refresh would still read the page-load + // bid out of ts.bids and claim the (long finished) server-side auction + // rendered it. + // + // The tuple is snapshotted here rather than re-read from `ts.bids` when + // the render event arrives: by then a newer navigation may have + // replaced `bids`, and stamping this render with that route's auction + // would both mislabel it and leave the real render of that auction + // demoted to `gam-refresh`. + const armed = TS_BID_TARGETING_KEYS.some((key) => Boolean(bid[key])); + enqueueGptRender(gptSlot, { + generation, + slotId: slot.id, + // Snapshot request data instead of reading a newer route's `ts.bids` + // when this request's render event eventually arrives. + bid: { ...bid }, + attributed: armed, + }); // Map both inner div and container div → slot ID so slotRenderEnded // (which reports the GPT slot's div, i.e. slotDivId/container) can look up // the slot, while adm injection (which targets the inner div) also works. @@ -634,34 +942,92 @@ export function installTsAdInit(): void { g.pubads!().addEventListener?.('slotRenderEnded', (event: SlotRenderEndedEvent) => { const divId: string = event.slot?.getSlotElementId?.() ?? ''; - const slotId = (ts.divToSlotId ?? {})[divId]; + const pending = takeGptRender(event.slot); + if (pending && pending.generation !== currentGeneration(ts)) { + log.debug('[tsjs-gpt] ignoring slotRenderEnded from a superseded route', { divId }); + return; + } + + const slotId = pending?.slotId ?? (ts.divToSlotId ?? {})[divId]; if (!slotId) return; - // Read ts.bids live (not the snapshot above) so post-navigation bid data is used. - const bid = (ts.bids ?? {})[slotId] ?? {}; - const record = recordRender({ - slotId, - path: 'ssat', + const bid = pending?.bid ?? (ts.bids ?? {})[slotId] ?? {}; + + // Trace: registry entry + DOM markers joining the GAM render to the + // server-side auction bid. `rendered` is GAM's own non-empty signal; + // `injected`/`visible` carry the honest "is the TS creative actually + // on screen" state so the panel does not overclaim (a non-empty GAM + // slot is not proof the TS creative rendered). + const slotEl = document.getElementById(divId); + const eventGeneration = currentGeneration(ts); + + // The server-side auction runs once per navigation. Only the render + // that consumes the targeting adInit just applied may be attributed + // to it; a later publisher refresh re-requests GAM on its own and the + // creative it returns has no traceable link to any TS auction, so no + // bid tuple is stamped. Single-shot: the armed tuple is consumed here. + const attributed = pending?.attributed + ? { + path: 'ssat' as const, + auctionId: bid.hb_auction_id, + bidder: bid.hb_bidder, + adId: bid.hb_adid, + bidId: bid.hb_bid_id, + creativeId: bid.hb_crid, + admHash: bid.hb_adm_hash, + } + : { path: 'gam-refresh' as const }; + + // The record this event belongs to, resolved below. Captured by the + // deferred-placement callback, which fires an animation frame later. + let record: RenderRecord | undefined; + + // GAM interceptor (testing): when adm is present, replace the GAM creative. + // Adapted from PR #241 — uses window.tsjs.bids[slotId].adm instead of pbjs. + // Only active when inject_adm_for_testing injects adm into bids server-side. + // Run before recording so the trace reflects the post-injection state. + // No adm to inject means TS only applied GAM targeting: whatever GAM + // rendered lives in a cross-origin iframe TS cannot read, so this is + // explicitly *not* a confirmed TS placement (status: gam-only). + const injected = + bid.adm && bid.debug_bid + ? injectAdmIntoSlot( + divId, + bid.adm, + (placed) => { + // The animation-frame retry has resolved. Promote the record + // unless a newer render already replaced this impression. + if (!placed || !record || ts.renders?.[slotId] !== record || !slotEl) return; + updateRender(record, { injected: true, visible: isEffectivelyVisible(slotEl) }); + stampCreativeTrace(slotEl, record); + }, + () => + currentGeneration(ts) === eventGeneration && + !!slotEl?.isConnected && + document.getElementById(divId) === slotEl + ) + : false; + + const gamSignal = { rendered: !event.isEmpty, + gamEmpty: event.isEmpty, + injected, + visible: isEffectivelyVisible(slotEl), elementId: divId, - auctionId: bid.hb_auction_id, - bidder: bid.hb_bidder, - adId: bid.hb_adid, - creativeId: bid.hb_crid, - admHash: bid.hb_adm_hash, - servedFrom: 'gam', - }); - const slotElement = document.getElementById(divId); - if (slotElement) stampCreativeTrace(slotElement, record); - - // GAM interceptor (testing bypass): directly replace the GAM creative. - // `adm` is now always injected in production, so it can no longer gate - // this path. `debug_bid` is present only when inject_adm_for_testing is - // on, so it is the per-bid signal that the testing bypass is enabled. - // In production the render bridge serves the creative and GAM stays in - // the loop; this direct replace stays testing-only. - if (bid.adm && bid.debug_bid) { - injectAdmIntoSlot(divId, bid.adm); + }; + // The bridge may already have opened this impression's record (it + // serves the creative the same GAM request asked for). Enrich that + // record rather than appending a second one for the same impression. + const open = event.isEmpty + ? undefined + : enrichableImpression(ts, slotId, bid.hb_adid, 'gam'); + if (open) { + open.gamSeen = true; + record = updateRender(open.record, gamSignal); + } else { + record = recordRender({ slotId, servedFrom: 'gam', ...gamSignal, ...attributed }); + openImpression(ts, slotId, bid.hb_adid, record, 'gam'); } + if (slotEl) stampCreativeTrace(slotEl, record); }); } @@ -698,6 +1064,12 @@ export function installTsAdInit(): void { ts.adInitRefreshInProgress = false; } } + + // Only TS-defined slots can be orphaned by a re-render — publisher-owned + // slots are theirs to manage, and TS never destroys them. + if (newSlots.length > 0) { + watchForOrphanedSlots(ts); + } }); }; } @@ -786,6 +1158,10 @@ export function installSpaAuctionHook(): void { if (path === currentPath) return; currentPath = path; inflight?.abort(); + // Retire the route being left before awaiting anything. The new DOM can + // commit long before page-bids answers, and until then every watcher and + // in-flight render still refers to the previous route's auction. + beginNavigation(ts); const controller = new AbortController(); inflight = controller; @@ -987,26 +1363,87 @@ export function parseCachedBid(body: string): CachedBid | undefined { * below drops the stamp when the captured element has left the document. */ function recordBridgeRender( + ts: TsjsApi, slotId: string, bid: AuctionBidData, servedFrom: RenderServedFrom, el: HTMLElement | null ): void { - const record = recordRender({ - slotId, - path: 'ssat', + // The bridge serves TS's own markup into the Prebid Universal Creative, so + // this is a confirmed TS placement (injected: true). + const bridgeSignal = { rendered: true, + injected: true, + visible: isEffectivelyVisible(el), elementId: el?.id, - auctionId: bid.hb_auction_id, - bidder: bid.hb_bidder, - adId: bid.hb_adid, - creativeId: bid.hb_crid, - admHash: bid.hb_adm_hash, servedFrom, - }); + }; + // `slotRenderEnded` may already have opened this impression's record for the + // same GAM request. Enrich it — appending here would count one creative as + // two renders, and this confirmed placement must not be a separate row from + // the GAM fill signal describing it. + const open = enrichableImpression(ts, slotId, bid.hb_adid, 'bridge'); + let record: RenderRecord; + if (open) { + open.bridgeSeen = true; + record = updateRender(open.record, bridgeSignal); + } else { + record = recordRender({ + slotId, + path: 'ssat', + auctionId: bid.hb_auction_id, + bidder: bid.hb_bidder, + adId: bid.hb_adid, + bidId: bid.hb_bid_id, + creativeId: bid.hb_crid, + admHash: bid.hb_adm_hash, + ...bridgeSignal, + }); + openImpression(ts, slotId, bid.hb_adid, record, 'bridge'); + } if (el && el.isConnected) stampCreativeTrace(el, record); } +/** + * Whether a PBS Cache result may still be applied when its fetch resolves. + * + * The fetch is started for one specific impression on one specific route, but + * settles arbitrarily later. By then an SPA navigation may have rebound the + * slot to a new auction — writing the record anyway would make a finished + * auction the slot's current render, fire a render event for a creative nobody + * can see, and stamp the new route's element with the old route's tuple. + * + * So all four anchors of that impression must still hold: the route it started + * in, the element it resolved to, the message source still living under that + * same slot, and the slot still showing the very bid it was fetched for. + */ +function bridgeResultStillCurrent( + ts: TsjsApi, + generation: number, + slotId: string, + bid: AuctionBidData, + el: HTMLElement | null, + source: MessageEventSource | null +): boolean { + if (currentGeneration(ts) !== generation) return false; + if (!el?.isConnected) return false; + if (slotIdForMessageSource(source) !== slotId) return false; + const live = (ts.bids ?? {})[slotId]; + if (!live) return false; + return ( + live.hb_adid === bid.hb_adid && + live.hb_auction_id === bid.hb_auction_id && + live.hb_bid_id === bid.hb_bid_id && + live.hb_bidder === bid.hb_bidder && + live.hb_crid === bid.hb_crid && + live.hb_adm_hash === bid.hb_adm_hash && + live.hb_cache_host === bid.hb_cache_host && + live.hb_cache_path === bid.hb_cache_path && + live.nurl === bid.nurl && + live.burl === bid.burl + ); +} + export function installTsRenderBridge(): void { if (typeof window === 'undefined') return; @@ -1132,20 +1569,26 @@ export function installTsRenderBridge(): void { // pulling slot B's creative and firing slot B's win/billing beacons. if (!matchedBid || matchedBid.hb_adid !== adId) return; - const slot = window.tsjs?.adSlots?.find((s) => s.id === slotId); + const ts = (window.tsjs ??= {} as TsjsApi); + // Snapshot every field the asynchronous cache completion will consume. + const capturedBid: AuctionBidData = { ...matchedBid }; + const slot = ts.adSlots?.find((s) => s.id === slotId); // Prefer the winning creative's own dimensions; the first configured slot // format is only a fallback and mis-sizes a multi-size slot whose winner is // not the first format. const [fallbackWidth, fallbackHeight] = slot?.formats?.[0] ?? [728, 90]; - const width = matchedBid.w ?? fallbackWidth; - const height = matchedBid.h ?? fallbackHeight; + const width = capturedBid.w ?? fallbackWidth; + const height = capturedBid.h ?? fallbackHeight; // Resolve the slot element now, at message-receipt time: the PBS Cache // branch stamps after an async fetch, and by then an SPA navigation may // have swapped tsjs.adSlots/DOM for a new route with the same slot IDs. const slotEl = slot ? findSlotElementByDivId(slot.div_id) : null; + // The route this render belongs to, re-checked when the async fetch below + // resolves. + const generation = currentGeneration(ts); - if (matchedBid.renderer !== undefined) { - const renderer = validateApsRenderer(matchedBid.renderer); + if (capturedBid.renderer !== undefined) { + const renderer = validateApsRenderer(capturedBid.renderer); const rendererUrl = apsRendererUrl(); if (!renderer || !rendererUrl) return; @@ -1177,25 +1620,26 @@ export function installTsRenderBridge(): void { return; } - if (matchedBid.adm) { + if (capturedBid.adm) { e.stopImmediatePropagation(); port.postMessage( JSON.stringify({ message: 'Prebid Response', adId, - ad: matchedBid.adm, + ad: capturedBid.adm, renderer: TS_DISPLAY_RENDERER, width, height, }) ); - fireWinBillingBeacons(slotId, matchedBid); + fireWinBillingBeacons(slotId, capturedBid); + recordBridgeRender(ts, slotId, capturedBid, 'debug-adm', slotEl); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from inline adm`); return; } // No TS render source — let Prebid.js handle it. - if (!matchedBid.hb_cache_host || !matchedBid.hb_cache_path) return; + if (!capturedBid.hb_cache_host || !capturedBid.hb_cache_path) return; // TS owns this adId — stop Prebid from also processing it. e.stopImmediatePropagation(); @@ -1206,11 +1650,20 @@ export function installTsRenderBridge(): void { if (renderingKeys.has(renderingKey)) return; renderingKeys.add(renderingKey); - const cacheUrl = `https://${matchedBid.hb_cache_host}${matchedBid.hb_cache_path}?uuid=${encodeURIComponent(adId)}`; + const cacheUrl = `https://${capturedBid.hb_cache_host}${capturedBid.hb_cache_path}?uuid=${encodeURIComponent(adId)}`; + + // Abortable so a navigation can cancel a render belonging to the route it + // is leaving instead of letting it land on the new one. + const controller = new AbortController(); + inflightCacheFetches.add(controller); - fetch(cacheUrl, { mode: 'cors' }) + fetch(cacheUrl, { mode: 'cors', signal: controller.signal }) .then((res) => (res.ok ? res.text() : Promise.reject(res.status))) .then((body) => { + if (!bridgeResultStillCurrent(ts, generation, slotId, capturedBid, slotEl, e.source)) { + log.debug(`[tsjs-gpt] pbRender bridge: dropping stale PBS Cache result for '${slotId}'`); + return; + } // PBS Cache returns the cached bid as a JSON object; decode its creative // and render metadata the same way the Prebid Universal Creative does. const cached = parseCachedBid(body); @@ -1239,15 +1692,21 @@ export function installTsRenderBridge(): void { height: cached.height ?? height, }) ); - fireWinBillingBeacons(slotId, matchedBid); - recordBridgeRender(slotId, matchedBid, 'pbs-cache', slotEl); + // Beacons carry the server-expanded ${AUCTION_PRICE} from the auction's + // clearing price, not `cached.price` — the auction result is the + // authoritative clearing price, and the cached copy is only the render + // source. Do not re-expand them here. + fireWinBillingBeacons(slotId, capturedBid); + recordBridgeRender(ts, slotId, capturedBid, 'pbs-cache', slotEl); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from PBS Cache`); }) .catch((err) => { + if (err instanceof DOMException && err.name === 'AbortError') return; log.warn(`[tsjs-gpt] pbRender bridge: PBS Cache fetch failed for '${slotId}'`, err); }) .finally(() => { renderingKeys.delete(renderingKey); + inflightCacheFetches.delete(controller); }); }); } diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 735334776..529793a4c 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -31,7 +31,8 @@ import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import { registerApsPrebidRenderer } from '../aps/render'; import type { AuctionBid, AuctionEid } from '../../core/auction'; -import type { AuctionSlot } from '../../core/types'; +import type { AuctionSlot, RenderRecord } from '../../core/types'; +import { recordRender, stampCreativeTrace, isEffectivelyVisible } from '../../core/trace'; import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated'; import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; @@ -226,6 +227,14 @@ export function auctionBidsToPrebidBids(auctionBids: AuctionBid[], bidRequests: bidderCode: bid.seat, meta: { advertiserDomains: bid.adomain, + // Carry the server-side trace tuple through Prebid so the render-trace + // hook can attribute the render to its /auction (see + // installPrebidRenderTrace). Prebid passes `meta` through unchanged. + // `tsBidId` is the bid's own OpenRTB `id`, kept separate from + // `creativeId` (the advertiser's `crid`, reused across bids). + tsAuctionId: bid.auctionId, + tsBidId: bid.bidId, + tsAdmHash: bid.admHash, }, }; }); @@ -1215,10 +1224,135 @@ function syncPrebidEidsCookie(): void { } } +// --------------------------------------------------------------------------- +// Render trace (client-side /auction path) +// --------------------------------------------------------------------------- + +/** Minimal shape of the bid object Prebid.js attaches to a render event. */ +interface PrebidRenderedBid { + adUnitCode?: string; + bidderCode?: string; + bidder?: string; + creativeId?: string; + meta?: { + tsAuctionId?: unknown; + tsBidId?: unknown; + tsAdmHash?: unknown; + [key: string]: unknown; + }; +} + +/** + * Payload of Prebid's `adRenderSucceeded` (`{ doc, bid, adId }`) and + * `adRenderFailed` (`{ reason, message, bid?, adId? }`) events. + */ +interface PrebidRenderEvent { + bid?: PrebidRenderedBid; + adId?: string; + reason?: string; + message?: string; +} + +/** + * Resolve the on-page element for a `bidWon` ad-unit code. Prebid renders into + * the ad unit's own div; fall back to the `-container` wrapper used by the GPT + * integration when the inner div is not directly addressable. + */ +function findAuctionSlotElement(adUnitCode: string): HTMLElement | null { + if (typeof document === 'undefined') return null; + return (document.getElementById(adUnitCode) ?? + document.getElementById(`${adUnitCode}-container`)) as HTMLElement | null; +} + +/** + * Record a render-trace entry for a completed Prebid render attempt on the + * client-side `/auction` path, distinct from the SSAT/GAM path. + * + * Only server-side (`trustedServer`) bids carry `meta.tsAuctionId` (set in + * {@link auctionBidsToPrebidBids}); client-side bidders lack it and are skipped + * so the panel never attributes a non-TS render to Trusted Server. + * + * `outcome` comes from which event fired. A failed attempt is still recorded — + * silence would read as "never won" — but as `rendered: false, injected: false`, + * which [`panelStatus`] reports as `empty`, so the panel shows a red row rather + * than a confirmed green one. + * + * Exported for unit testing; the render path itself is unaffected (this only + * observes and stamps the DOM). + */ +export function recordPrebidAdRender( + bid: PrebidRenderedBid | undefined, + outcome: 'succeeded' | 'failed' +): RenderRecord | undefined { + if (!bid || typeof bid.adUnitCode !== 'string' || bid.adUnitCode === '') return undefined; + const meta = bid.meta ?? {}; + // Only trace our server-side bids: the trustedServer adapter is the only + // source of meta.tsAuctionId. + if (typeof meta.tsAuctionId !== 'string') return undefined; + + const succeeded = outcome === 'succeeded'; + const el = findAuctionSlotElement(bid.adUnitCode); + const record = recordRender({ + slotId: bid.adUnitCode, + path: 'auction', + rendered: succeeded, + // Prebid rendered the `ad` markup our adapter returned — a confirmed TS + // placement for this path, but only once the render actually succeeded. + injected: succeeded, + visible: succeeded ? isEffectivelyVisible(el) : false, + elementId: el?.id, + auctionId: meta.tsAuctionId, + bidId: typeof meta.tsBidId === 'string' ? meta.tsBidId : undefined, + admHash: typeof meta.tsAdmHash === 'string' ? meta.tsAdmHash : undefined, + bidder: bid.bidderCode ?? bid.bidder, + creativeId: bid.creativeId, + servedFrom: 'prebid', + }); + if (el) stampCreativeTrace(el, record); + return record; +} + +/** + * Install the Prebid render-trace hooks (idempotent). + * + * Listens on `adRenderSucceeded` / `adRenderFailed`, **not** `bidWon`. `bidWon` + * fires when Prebid marks a bid as the auction winner — before it hands the bid + * to a renderer that can still fail asynchronously (`emitAdRenderFail`), so a + * `bidWon` listener would stamp the DOM and light up a confirmed green render + * for a creative that never reached the page. `adRenderSucceeded` is emitted + * only after the render function returned without error, and carries the + * rendered bid on `event.bid`. + * + * Without these hooks only the one-time SSAT auction is traced and the ongoing + * Prebid renders are invisible to the panel. Read-only: the listeners record and + * stamp but never alter Prebid's rendering. + */ +export function installPrebidRenderTrace(): void { + if (typeof window === 'undefined') return; + const p = pbjs as unknown as { + onEvent?: (event: string, handler: (event: PrebidRenderEvent) => void) => void; + __tsRenderTraceInstalled?: boolean; + }; + if (typeof p.onEvent !== 'function' || p.__tsRenderTraceInstalled) return; + p.__tsRenderTraceInstalled = true; + const listen = (name: string, outcome: 'succeeded' | 'failed'): void => { + p.onEvent!(name, (event) => { + try { + recordPrebidAdRender(event?.bid, outcome); + } catch (err) { + log.warn(`[tsjs-prebid] render-trace ${name} failed`, err); + } + }); + }; + listen('adRenderSucceeded', 'succeeded'); + listen('adRenderFailed', 'failed'); +} + // Self-initialize when loaded in a browser (same pattern as other integrations). if (typeof window !== 'undefined') { installPrebidNpm(); installRefreshHandler(); + installPrebidRenderTrace(); // The slim-Prebid lazy loader appends this bundle from a window.load // handler, so `load` may already have fired by the time this code runs — // waiting for it again would skip user ID setup entirely on that path. diff --git a/crates/trusted-server-js/lib/test/core/auction.test.ts b/crates/trusted-server-js/lib/test/core/auction.test.ts index 276f819a5..098deeb64 100644 --- a/crates/trusted-server-js/lib/test/core/auction.test.ts +++ b/crates/trusted-server-js/lib/test/core/auction.test.ts @@ -332,6 +332,7 @@ describe('auction/parseAuctionResponse', () => { seat: 'kargo', bid: [ { + id: 'bid-uuid-7', impid: 'slot-1', price: 1.0, adm: '

A
', @@ -344,6 +345,7 @@ describe('auction/parseAuctionResponse', () => { const bids = parseAuctionResponse(body); expect(bids[0].auctionId).toBe('auction-uuid-2'); + expect(bids[0].bidId).toBe('bid-uuid-7'); expect(bids[0].admHash).toBe('a1b2c3d4e5f60718'); }); }); diff --git a/crates/trusted-server-js/lib/test/core/trace.test.ts b/crates/trusted-server-js/lib/test/core/trace.test.ts index ba7a640e2..2c737099e 100644 --- a/crates/trusted-server-js/lib/test/core/trace.test.ts +++ b/crates/trusted-server-js/lib/test/core/trace.test.ts @@ -1,10 +1,30 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { recordRender, stampCreativeTrace, RENDER_EVENT_NAME } from '../../src/core/trace'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import { + recordRender, + updateRender, + stampCreativeTrace, + traceOverlayEnabled, + renderTracePanel, + RENDER_EVENT_NAME, + TRACE_PANEL_ID, + TRACE_BADGE_CLASS, +} from '../../src/core/trace'; import type { RenderRecord, TsjsApi } from '../../src/core/types'; +function clearTraceCookie(): void { + document.cookie = 'ts-trace=; Max-Age=0; Path=/'; +} + +function removePanel(): void { + document.getElementById(TRACE_PANEL_ID)?.remove(); +} + describe('trace/recordRender', () => { beforeEach(() => { delete (window as { tsjs?: TsjsApi }).tsjs; + clearTraceCookie(); + removePanel(); }); it('writes a render record into window.tsjs.renders', () => { @@ -52,6 +72,68 @@ describe('trace/recordRender', () => { window.removeEventListener(RENDER_EVENT_NAME, listener); }); + + it('allocates one sequence across separately bundled IIFEs', async () => { + const buildTraceBundle = (): string => + execFileSync( + './node_modules/.bin/esbuild', + ['--bundle', '--format=iife', '--platform=browser', '--loader=ts'], + { + cwd: process.cwd(), + encoding: 'utf8', + input: + 'import { recordRender } from "./src/core/trace.ts";' + + 'window.__recordFromTraceBundle = recordRender;', + } + ); + + const firstBundle = buildTraceBundle(); + const secondBundle = buildTraceBundle(); + const testWindow = window as typeof window & { + __recordFromTraceBundle?: typeof recordRender; + }; + + Function(firstBundle)(); + const firstRecord = testWindow.__recordFromTraceBundle!; + const first = firstRecord({ slotId: 'iife-a', path: 'auction', rendered: true }); + + Function(secondBundle)(); + const secondRecord = testWindow.__recordFromTraceBundle!; + const second = secondRecord({ slotId: 'iife-b', path: 'ssat', rendered: true }); + + expect(second.seq).toBe(first.seq + 1); + expect(window.tsjs?.renderSeq).toBe(second.seq); + delete testWindow.__recordFromTraceBundle; + }); + + it('enriches an existing impression without changing its bookkeeping', () => { + const original = recordRender({ + slotId: 'slot-enrich', + path: 'ssat', + rendered: true, + injected: false, + servedFrom: 'gam', + }); + const bookkeeping = { + seq: original.seq, + count: original.count, + at: original.at, + historyLength: window.tsjs?.renderLog?.length, + }; + + const updated = updateRender(original, { injected: true, servedFrom: 'pbs-cache' }); + + expect(updated).toBe(original); + expect(window.tsjs?.renders?.['slot-enrich']).toBe(original); + expect(window.tsjs?.renderLog?.[0]).toBe(original); + expect(updated).toEqual(expect.objectContaining({ injected: true, servedFrom: 'pbs-cache' })); + expect({ + seq: updated.seq, + count: updated.count, + at: updated.at, + historyLength: window.tsjs?.renderLog?.length, + }).toEqual(bookkeeping); + }); }); describe('trace/stampCreativeTrace', () => { @@ -66,6 +148,7 @@ describe('trace/stampCreativeTrace', () => { adId: 'cache-uuid-1', admHash: 'a1b2c3d4e5f60718', count: 1, + seq: 1, at: 1, }; @@ -92,6 +175,7 @@ describe('trace/stampCreativeTrace', () => { admHash: 'a1b2c3d4e5f60718', servedFrom: 'gam', count: 1, + seq: 1, at: 1, }; stampCreativeTrace(el, first); @@ -102,6 +186,7 @@ describe('trace/stampCreativeTrace', () => { rendered: true, auctionId: 'auction-new', count: 2, + seq: 2, at: 2, }; stampCreativeTrace(el, second); @@ -112,3 +197,317 @@ describe('trace/stampCreativeTrace', () => { expect(el.hasAttribute('data-ts-served-from')).toBe(false); }); }); + +describe('trace/floating panel', () => { + const record: Omit = { + slotId: 'slot-1', + path: 'ssat', + rendered: true, + injected: true, + visible: true, + gamEmpty: false, + auctionId: 'ts-req-abcdef123456', + bidder: 'kargo', + admHash: 'a1b2c3d4e5f60718', + servedFrom: 'gam', + }; + + beforeEach(() => { + delete (window as { tsjs?: TsjsApi }).tsjs; + clearTraceCookie(); + removePanel(); + }); + + afterEach(() => { + clearTraceCookie(); + removePanel(); + }); + + it('reports the overlay disabled without the ts-trace cookie', () => { + expect(traceOverlayEnabled()).toBe(false); + }); + + it('does not create a panel when the overlay is disarmed', () => { + recordRender(record); + expect(document.getElementById(TRACE_PANEL_ID)).toBeNull(); + }); + + it('renders a panel row per traced slot with honest status', () => { + document.cookie = 'ts-trace=1; Path=/'; + // slot-1: TS placed + visible → ok. slot-2: nothing rendered → empty. + recordRender(record); + recordRender({ + slotId: 'slot-2', + path: 'auction', + rendered: false, + injected: false, + visible: false, + bidder: 'appnexus', + }); + + const panel = document.getElementById(TRACE_PANEL_ID); + expect(panel).toBeTruthy(); + // Only slot-1 is honestly ok; slot-2 rendered nothing. + expect(panel!.textContent).toContain('TS Render Trace · 1/2 slots ok'); + expect(panel!.textContent).toContain('✓ slot-1 · ok'); + expect(panel!.textContent).toContain('✗ slot-2 · empty'); + expect(panel!.textContent).toContain('ssat · kargo'); + expect(panel!.textContent).toContain('auction · appnexus'); + }); + + it('marks a rendered-but-hidden slot as hidden, not ok', () => { + document.cookie = 'ts-trace=1; Path=/'; + // GAM rendered non-empty, TS injected, but a reveal gate keeps it hidden. + recordRender({ ...record, visible: false }); + + const panel = document.getElementById(TRACE_PANEL_ID); + expect(panel!.textContent).toContain('0/1 slots ok'); + expect(panel!.textContent).toContain('⚠ slot-1 · hidden'); + }); + + it('marks a targeting-only GAM slot as gam-only, not a confirmed TS render', () => { + document.cookie = 'ts-trace=1; Path=/'; + // GAM rendered something, but TS never placed it (prod targeting path). + recordRender({ ...record, injected: false, gamEmpty: false, visible: true }); + + const panel = document.getElementById(TRACE_PANEL_ID); + expect(panel!.textContent).toContain('0/1 slots ok'); + expect(panel!.textContent).toContain('◐ slot-1 · gam-only'); + }); + + it('never claims ok when a render path did not report placement', () => { + document.cookie = 'ts-trace=1; Path=/'; + // Regression: an unset `injected` must not fall through to ok — that would + // claim a confirmed TS render for a slot TS only targeted. + const { injected: _omitted, ...withoutInjected } = record; + recordRender({ ...withoutInjected, gamEmpty: false, visible: true }); + + const panel = document.getElementById(TRACE_PANEL_ID); + expect(panel!.textContent).toContain('0/1 slots ok'); + expect(panel!.textContent).toContain('◐ slot-1 · gam-only'); + }); + + it('reuses a single panel across renders and reflects the latest count', () => { + document.cookie = 'ts-trace=1; Path=/'; + recordRender(record); + recordRender(record); + + const panels = document.querySelectorAll(`#${TRACE_PANEL_ID}`); + expect(panels).toHaveLength(1); + // Second render of the same slot bumps the count and appends a history row. + expect(panels[0].textContent).toContain('TS Render Trace · 1/1 slots ok'); + expect(panels[0].textContent).toContain('×2'); + }); + + it("keeps GAM's fill signal and drops ? placeholders on an unattributed refresh", () => { + document.cookie = 'ts-trace=1; Path=/'; + // A publisher-driven GAM refresh: TS ran no auction for it, so there is no + // bidder/hash/auction id — but GAM still reported whether it filled, and + // that is the most useful field on the row. + recordRender({ + slotId: 'slot-1', + path: 'gam-refresh', + rendered: true, + gamEmpty: false, + injected: false, + visible: true, + servedFrom: 'gam', + }); + + const panel = document.getElementById(TRACE_PANEL_ID)!; + expect(panel.textContent).toContain('gam:filled'); + expect(panel.textContent).toContain('no TS attribution'); + // Absent attribution must not render as a failed lookup, and an auction + // segment must not appear at all when there is no auction to name. + expect(panel.textContent).not.toContain('· ? ·'); + expect(panel.textContent).not.toContain('auction ?'); + }); + + it('still reports gam:empty for a refresh GAM declined to fill', () => { + document.cookie = 'ts-trace=1; Path=/'; + recordRender({ + slotId: 'slot-1', + path: 'gam-refresh', + rendered: false, + gamEmpty: true, + injected: false, + visible: true, + }); + + const panel = document.getElementById(TRACE_PANEL_ID)!; + expect(panel.textContent).toContain('gam:empty'); + expect(panel.textContent).toContain('✗ slot-1 · empty'); + }); + + it('gives each render a page-global seq the badge and its panel row share', () => { + document.cookie = 'ts-trace=1; Path=/'; + const el = document.createElement('div'); + el.id = 'slot-el'; + document.body.appendChild(el); + + // Two slots interleaved: seq must be unique page-wide, not per-slot, so a + // badge reading #N identifies exactly one row. + const first = recordRender(record); + const other = recordRender({ ...record, slotId: 'slot-2' }); + expect(other.seq).toBe(first.seq + 1); + + stampCreativeTrace(el, other); + const badge = el.querySelector(`.${TRACE_BADGE_CLASS}`) as HTMLElement; + expect(badge.textContent).toContain(`#${other.seq}`); + // The same number appears on that render's row in the panel. + expect(document.getElementById(TRACE_PANEL_ID)!.textContent).toContain(`#${other.seq}`); + + el.remove(); + }); + + it('marks only the live render for a slot as current', () => { + document.cookie = 'ts-trace=1; Path=/'; + recordRender(record); + const latest = recordRender(record); + + const panel = document.getElementById(TRACE_PANEL_ID)!; + // Both renders are in the log, but only the newest is still on screen. + expect(panel.textContent).toContain(`#${latest.seq}`); + expect(panel.textContent!.match(/◂ current/g)).toHaveLength(1); + }); + + it('uses record identity when duplicate sequence values exist', () => { + document.cookie = 'ts-trace=1; Path=/'; + const oldRecord = { ...record, auctionId: 'auction-old', count: 1, seq: 7, at: 1 }; + const liveRecord = { ...record, auctionId: 'auction-live', count: 2, seq: 7, at: 2 }; + (window as { tsjs?: TsjsApi }).tsjs = { + renders: { 'slot-1': liveRecord }, + renderLog: [oldRecord, liveRecord], + } as unknown as TsjsApi; + + renderTracePanel(); + + const rows = [...document.querySelectorAll(`#${TRACE_PANEL_ID} div[style*="cursor"]`)]; + const oldRow = rows.find((row) => row.getAttribute('title')?.includes('auction: auction-old')); + const liveRow = rows.find((row) => + row.getAttribute('title')?.includes('auction: auction-live') + ); + expect(oldRow?.textContent).not.toContain('◂ current'); + expect(liveRow?.textContent).toContain('◂ current'); + }); + + it('close button removes the panel', () => { + document.cookie = 'ts-trace=1; Path=/'; + recordRender(record); + const panel = document.getElementById(TRACE_PANEL_ID)!; + const close = panel.querySelector('button') as HTMLButtonElement; + close.click(); + expect(document.getElementById(TRACE_PANEL_ID)).toBeNull(); + }); + + it('renderTracePanel is a no-op while disarmed even if renders exist', () => { + (window as { tsjs?: TsjsApi }).tsjs = { + renders: { 'slot-1': { ...record, count: 1, seq: 1, at: 1 } }, + } as unknown as TsjsApi; + renderTracePanel(); + expect(document.getElementById(TRACE_PANEL_ID)).toBeNull(); + }); + + it('clicking a row logs the full record', async () => { + document.cookie = 'ts-trace=1; Path=/'; + const { log } = await import('../../src/core/log'); + const infoSpy = vi.spyOn(log, 'info').mockImplementation(() => undefined); + + recordRender(record); + const row = document.getElementById(TRACE_PANEL_ID)!.querySelector('div[style*="cursor"]'); + (row as HTMLElement).click(); + + const call = infoSpy.mock.calls.find(([m]) => m === 'trace: render record'); + expect(call?.[1]).toEqual( + expect.objectContaining({ slotId: 'slot-1', auctionId: record.auctionId }) + ); + infoSpy.mockRestore(); + }); +}); + +describe('trace/confirmation badge', () => { + beforeEach(() => { + delete (window as { tsjs?: TsjsApi }).tsjs; + clearTraceCookie(); + document.body.innerHTML = ''; + }); + afterEach(() => { + clearTraceCookie(); + document.body.innerHTML = ''; + }); + + const okRecord: RenderRecord = { + slotId: 'slot-1', + path: 'ssat', + rendered: true, + injected: true, + visible: true, + gamEmpty: false, + bidder: 'mocktioneer', + admHash: 'a1b2c3d4e5f60718', + servedFrom: 'gam', + count: 1, + seq: 1, + at: 1, + }; + + it('badges an ok slot when armed', () => { + document.cookie = 'ts-trace=1; Path=/'; + const el = document.createElement('div'); + document.body.appendChild(el); + stampCreativeTrace(el, okRecord); + const badge = el.querySelector(`.${TRACE_BADGE_CLASS}`) as HTMLElement; + expect(badge).toBeTruthy(); + expect(badge.textContent).toBe('TS ✓ #1 · mocktioneer'); + }); + + it('does not badge a hidden slot', () => { + document.cookie = 'ts-trace=1; Path=/'; + const el = document.createElement('div'); + document.body.appendChild(el); + stampCreativeTrace(el, { ...okRecord, visible: false }); + expect(el.querySelector(`.${TRACE_BADGE_CLASS}`)).toBeNull(); + }); + + it('removes the previous badge when a filled slot becomes empty', () => { + document.cookie = 'ts-trace=1; Path=/'; + const el = document.createElement('div'); + document.body.appendChild(el); + stampCreativeTrace(el, okRecord); + expect(el.querySelector(`.${TRACE_BADGE_CLASS}`)).toBeTruthy(); + + stampCreativeTrace(el, { ...okRecord, rendered: false, injected: false, gamEmpty: true }); + + expect(el.querySelector(`.${TRACE_BADGE_CLASS}`)).toBeNull(); + }); + + it('removes the previous badge when a visible slot becomes hidden', () => { + document.cookie = 'ts-trace=1; Path=/'; + const el = document.createElement('div'); + document.body.appendChild(el); + stampCreativeTrace(el, okRecord); + expect(el.querySelector(`.${TRACE_BADGE_CLASS}`)).toBeTruthy(); + + stampCreativeTrace(el, { ...okRecord, visible: false }); + + expect(el.querySelector(`.${TRACE_BADGE_CLASS}`)).toBeNull(); + }); + + it('does not badge when the overlay is disarmed', () => { + clearTraceCookie(); + const el = document.createElement('div'); + document.body.appendChild(el); + stampCreativeTrace(el, okRecord); + expect(el.querySelector(`.${TRACE_BADGE_CLASS}`)).toBeNull(); + }); + + it('never badges an iframe element', () => { + document.cookie = 'ts-trace=1; Path=/'; + const iframe = document.createElement('iframe'); + document.body.appendChild(iframe); + stampCreativeTrace(iframe, okRecord); + expect(iframe.querySelector(`.${TRACE_BADGE_CLASS}`)).toBeNull(); + // Attributes still stamped on the iframe though. + expect(iframe.getAttribute('data-ts-slot-id')).toBe('slot-1'); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 3d7dfa978..609fb86fd 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -673,6 +673,7 @@ describe('installTsAdInit', () => { hb_pb: '1.00', hb_bidder: 'kargo', hb_adid: 'cache-uuid-9', + hb_bid_id: 'bid-uuid-9', hb_auction_id: 'ts-req-trace9', hb_crid: 'cr-98765', hb_adm_hash: 'a1b2c3d4e5f60718', @@ -694,6 +695,7 @@ describe('installTsAdInit', () => { expect(el.getAttribute('data-ts-auction-id')).toBe('ts-req-trace9'); expect(el.getAttribute('data-ts-bidder')).toBe('kargo'); expect(el.getAttribute('data-ts-ad-id')).toBe('cache-uuid-9'); + expect(el.getAttribute('data-ts-bid-id')).toBe('bid-uuid-9'); expect(el.getAttribute('data-ts-creative-id')).toBe('cr-98765'); expect(el.getAttribute('data-ts-adm-hash')).toBe('a1b2c3d4e5f60718'); @@ -707,6 +709,7 @@ describe('installTsAdInit', () => { auctionId: 'ts-req-trace9', bidder: 'kargo', adId: 'cache-uuid-9', + bidId: 'bid-uuid-9', creativeId: 'cr-98765', admHash: 'a1b2c3d4e5f60718', servedFrom: 'gam', @@ -721,6 +724,309 @@ describe('installTsAdInit', () => { expect(el.getAttribute('data-ts-rendered')).toBe('false'); }); + it('drops a late render from a reused publisher slot without consuming the new route', async () => { + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: { + atf_sidebar_ad: { + hb_bidder: 'route-a', + hb_adid: 'ad-a', + hb_bid_id: 'bid-a', + hb_auction_id: 'auction-a', + }, + }, + }; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + const ts = (window as TestWindow).tsjs!; + ts.adInit!(); + + ts.bids = { + atf_sidebar_ad: { + hb_bidder: 'route-b', + hb_adid: 'ad-b', + hb_bid_id: 'bid-b', + hb_auction_id: 'auction-b', + }, + }; + ts.adInit!(); + + // Route A's event arrives only after route B has reused and refreshed the + // same publisher-owned GPT slot object. It must be rejected, not stamped + // with route B's tuple or allowed to consume route B's pending request. + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect(ts.renders?.['atf_sidebar_ad']).toBeUndefined(); + + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect(ts.renders?.['atf_sidebar_ad']).toEqual( + expect.objectContaining({ + path: 'ssat', + auctionId: 'auction-b', + bidId: 'bid-b', + bidder: 'route-b', + }) + ); + }); + + it('does not confirm a deferred ADM placement after navigation supersedes it', async () => { + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + let runDeferredPlacement: FrameRequestCallback | undefined; + vi.stubGlobal( + 'requestAnimationFrame', + vi.fn((callback: FrameRequestCallback) => { + runDeferredPlacement = callback; + return 1; + }) + ); + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: { + atf_sidebar_ad: { + hb_bidder: 'mocktioneer', + hb_adid: 'deferred-ad', + hb_auction_id: 'auction-a', + adm: '', + debug_bid: {}, + }, + }, + }; + + try { + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + const ts = (window as TestWindow).tsjs!; + ts.adInit!(); + capturedListener!({ isEmpty: false, slot: mockSlot }); + + const oldRecord = ts.renders?.['atf_sidebar_ad']; + expect(oldRecord?.injected).toBe(false); + expect(runDeferredPlacement).toBeDefined(); + + // A newer route/adInit starts before the animation-frame retry runs while + // retaining the same publisher-owned slot element. The old callback must + // not mutate that shared element before its trace guard runs. + ts.adInit!(); + const reusedSlot = document.getElementById('div-atf-sidebar')!; + runDeferredPlacement!(0); + + expect(oldRecord?.injected).toBe(false); + expect(reusedSlot.querySelector('iframe')).toBeNull(); + expect(reusedSlot.getAttribute('data-ts-injected')).toBe('false'); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('confirms a successful deferred ADM placement on the original record', async () => { + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + let runDeferredPlacement: FrameRequestCallback | undefined; + vi.stubGlobal( + 'requestAnimationFrame', + vi.fn((callback: FrameRequestCallback) => { + runDeferredPlacement = callback; + return 1; + }) + ); + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: { + atf_sidebar_ad: { + hb_adid: 'deferred-ad', + adm: '', + debug_bid: {}, + }, + }, + }; + + try { + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + const ts = (window as TestWindow).tsjs!; + ts.adInit!(); + capturedListener!({ isEmpty: false, slot: mockSlot }); + const record = ts.renders?.atf_sidebar_ad; + const originalBookkeeping = { + seq: record?.seq, + count: record?.count, + at: record?.at, + historyLength: ts.renderLog?.length, + }; + + runDeferredPlacement!(0); + + expect(ts.renders?.atf_sidebar_ad).toBe(record); + expect(record?.injected).toBe(true); + expect({ + seq: record?.seq, + count: record?.count, + at: record?.at, + historyLength: ts.renderLog?.length, + }).toEqual(originalBookkeeping); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('does not attribute a later GAM refresh to the finished server-side auction', async () => { + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: { + atf_sidebar_ad: { + hb_bidder: 'kargo', + hb_adid: 'cache-uuid-9', + hb_auction_id: 'ts-req-trace9', + hb_adm_hash: 'a1b2c3d4e5f60718', + }, + }, + }; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + // First render consumes the targeting adInit applied → attributable. + capturedListener!({ isEmpty: false, slot: mockSlot }); + expect((window as TestWindow).tsjs!.renders?.['atf_sidebar_ad']?.path).toBe('ssat'); + + // A publisher-driven refresh fills the slot again, but the server-side + // auction ran once and is long finished. ts.bids still holds its data — + // re-stamping it would claim a render that auction never produced. + capturedListener!({ isEmpty: false, slot: mockSlot }); + + const refreshed = (window as TestWindow).tsjs!.renders?.['atf_sidebar_ad']; + expect(refreshed?.path).toBe('gam-refresh'); + expect(refreshed?.rendered).toBe(true); + expect(refreshed?.count).toBe(2); + expect(refreshed?.auctionId).toBeUndefined(); + expect(refreshed?.bidder).toBeUndefined(); + expect(refreshed?.admHash).toBeUndefined(); + + // Stale attribution must not survive on the DOM either. + const el = document.getElementById('div-atf-sidebar')!; + expect(el.getAttribute('data-ts-render-path')).toBe('gam-refresh'); + expect(el.hasAttribute('data-ts-auction-id')).toBe(false); + expect(el.hasAttribute('data-ts-adm-hash')).toBe(false); + }); + it('does not fire beacons for an APS-style bid that carries no hb_adid', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); let capturedListener: ((e: SlotRenderEvent) => void) | undefined; @@ -1513,7 +1819,9 @@ describe('installTsRenderBridge', () => { expect(fetchStub).toHaveBeenCalledWith( 'https://openads.example.com/cache?uuid=test-cache-uuid', - { mode: 'cors' } + // Carries an abort signal so a navigation can cancel a render belonging + // to the route it is leaving. + { mode: 'cors', signal: expect.any(AbortSignal) } ); expect(stopSpy).toHaveBeenCalled(); expect(portMessages).toHaveLength(1); @@ -1526,6 +1834,20 @@ describe('installTsRenderBridge', () => { expect(beaconSpy).toHaveBeenCalledWith('https://ssp.example/bill'); expect(beaconSpy).toHaveBeenCalledTimes(2); + // The PBS Cache branch must trace the same as the debug-adm branch — + // regression coverage for the two recordBridgeRender call sites staying + // in sync (this branch's stamp only lands after the async fetch settles). + const record = (window as TestWindow).tsjs!.renders?.['homepage_header']; + expect(record).toEqual( + expect.objectContaining({ + slotId: 'homepage_header', + path: 'ssat', + rendered: true, + injected: true, + servedFrom: 'pbs-cache', + }) + ); + bridgeListener!( Object.assign(new Event('message'), { data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), @@ -1675,6 +1997,80 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('keeps GAM and bridge signals for both arrival orders on one record per impression', async () => { + const source = createTrustedSlotIframe(); + let slotRenderListener: ((event: SlotRenderEvent) => void) | undefined; + const gptSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-header'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([gptSlot]), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, listener: (event: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') slotRenderListener = listener; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(gptSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + const ts = (window as TestWindow).tsjs!; + ts.bids.homepage_header = { + hb_adid: 'debug-first', + hb_bidder: 'mocktioneer', + }; + const bridgeListener = await captureBridgeListener(); + ts.adInit!(); + + const sendBridgeRequest = (adId: string): void => { + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId }), + ports: [{ postMessage: vi.fn() }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + }; + + // GAM first, bridge second. + slotRenderListener!({ isEmpty: false, slot: gptSlot }); + const firstRecord = ts.renders?.homepage_header; + ts.bids.homepage_header = { + ...ts.bids.homepage_header, + adm: '
First creative
', + }; + sendBridgeRequest('debug-first'); + expect(ts.renders?.homepage_header).toBe(firstRecord); + expect(firstRecord).toEqual( + expect.objectContaining({ count: 1, injected: true, servedFrom: 'debug-adm' }) + ); + expect(ts.renderLog).toHaveLength(1); + + // Bridge first, GAM second for the next impression. + ts.bids.homepage_header = { + hb_adid: 'debug-second', + hb_bidder: 'mocktioneer', + adm: '', + }; + ts.adInit!(); + sendBridgeRequest('debug-second'); + const secondRecord = ts.renders?.homepage_header; + slotRenderListener!({ isEmpty: false, slot: gptSlot }); + expect(ts.renders?.homepage_header).toBe(secondRecord); + expect(secondRecord).toEqual( + expect.objectContaining({ count: 2, injected: true, gamEmpty: false }) + ); + expect(ts.renderLog).toHaveLength(2); + }); + it('fetches PBS Cache once when two same-adId messages race before the fetch resolves', async () => { // Concurrent render double-fire guard: two 'Prebid Request' messages for the // same adId can arrive before the first cache fetch settles. The in-flight @@ -1728,6 +2124,47 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('drops a PBS Cache result when the live bid changed before fetch completion', async () => { + let resolveFetch: (value: Response) => void = () => {}; + fetchStub.mockReturnValue( + new Promise((resolve) => { + resolveFetch = resolve; + }) + ); + const ts = (window as TestWindow).tsjs!; + ts.bids.homepage_header = { + ...ts.bids.homepage_header, + hb_auction_id: 'auction-1', + hb_bid_id: 'bid-1', + }; + const bridgeListener = await captureBridgeListener(); + const portMessages: string[] = []; + const source = createTrustedSlotIframe(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + // Same bridge ad ID and auction ID, but a different bid object/trace ID. + // Comparing only hb_adid + hb_auction_id would incorrectly accept the old + // creative and stamp it with the captured route's data. + ts.bids.homepage_header = { + ...ts.bids.homepage_header, + hb_bid_id: 'bid-2', + hb_adm_hash: 'new-creative-hash', + }; + resolveFetch({ ok: true, text: () => Promise.resolve('
Old creative
') } as Response); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(portMessages).toHaveLength(0); + expect(ts.renders?.homepage_header).toBeUndefined(); + }); + it('does not let one slot block a PBS Cache render for another slot sharing an adId', async () => { // The in-flight guard must be scoped to the requesting slot, not the shared // adId: two distinct slots sharing one hb_adid must each fetch and render. @@ -1879,6 +2316,21 @@ describe('installTsRenderBridge', () => { expect(beaconSpy).toHaveBeenCalledWith('https://debug.example/bill'); expect(beaconSpy).toHaveBeenCalledTimes(2); beaconSpy.mockRestore(); + + // Serving debug adm through the Universal Creative bridge is a confirmed + // TS placement — same as the PBS Cache branch — and must not be silently + // untraced just because no cache fetch was involved. + const record = (window as TestWindow).tsjs!.renders?.['homepage_header']; + expect(record).toEqual( + expect.objectContaining({ + slotId: 'homepage_header', + path: 'ssat', + rendered: true, + injected: true, + bidder: 'mocktioneer', + servedFrom: 'debug-adm', + }) + ); }); it('sizes the inline response from the winning bid, not the first slot format', async () => { @@ -2205,3 +2657,106 @@ describe('installTsRenderBridge', () => { expect(fetchStub).not.toHaveBeenCalled(); }); }); + +describe('orphaned TS slot recovery', () => { + type TestWin = Window & { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + tsjs?: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + googletag?: any; + }; + + beforeEach(() => { + vi.resetModules(); + const tw = window as TestWin; + delete tw.tsjs; + delete tw.googletag; + document.body.innerHTML = ''; + }); + + afterEach(() => { + document.body.innerHTML = ''; + }); + + function slotStub(elementId: string) { + return { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue(elementId), + getTargeting: vi.fn().mockReturnValue([]), + }; + } + + it('reports slots whose bound element left the document', async () => { + const { orphanedTsSlots } = await import('../../../src/integrations/gpt/index'); + document.body.innerHTML = '
'; + + const live = slotStub('live-div'); + const orphan = slotStub('ad-header-0-_R_ssr_'); + const ts = { prevGptSlots: [live, orphan] }; + + const orphans = orphanedTsSlots(ts as never); + expect(orphans).toHaveLength(1); + expect(orphans[0].getSlotElementId()).toBe('ad-header-0-_R_ssr_'); + }); + + it('returns nothing when every TS slot still has its element', async () => { + const { orphanedTsSlots } = await import('../../../src/integrations/gpt/index'); + document.body.innerHTML = '
'; + const ts = { prevGptSlots: [slotStub('a'), slotStub('b')] }; + expect(orphanedTsSlots(ts as never)).toHaveLength(0); + }); + + it('re-runs adInit after a re-render swaps the ad div', async () => { + // SSR div that hydration will replace. + document.body.innerHTML = '
'; + + const definedSlots: string[] = []; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + refresh: vi.fn(), + addEventListener: vi.fn(), + }; + const tw = window as TestWin; + tw.googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn((_path: string, _sizes: unknown, divId: string) => { + definedSlots.push(divId); + return slotStub(divId); + }), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + display: vi.fn(), + destroySlots: vi.fn(), + }; + tw.tsjs = { + adSlots: [ + { + id: 'ad-header-0', + gam_unit_path: '/123/header', + div_id: 'ad-header-0', + formats: [[728, 90]], + targeting: {}, + }, + ], + bids: {}, + }; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + tw.tsjs.adInit(); + + expect(definedSlots).toEqual(['ad-header-0-_R_ssr_']); + + // Hydration: React replaces the SSR div with a client-id div. + document.body.innerHTML = '
'; + + // The MutationObserver is debounced; give it room to fire. + await new Promise((r) => setTimeout(r, 600)); + + // adInit re-ran and bound to the live div instead of the dead one. + expect(definedSlots).toContain('ad-header-0-_r_1_'); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 761fa8b3b..35cfd14b1 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -51,6 +51,7 @@ describe('installSpaAuctionHook', () => { originalReplaceState({}, '', '/'); // Drop any ad containers inserted by a test so DOM state does not leak. document.body.innerHTML = ''; + delete (window as TestWindow).googletag; // Remove this test's popstate listener(s) so they do not fire in later tests. popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler)); popstateHandlers = []; @@ -304,6 +305,59 @@ describe('installSpaAuctionHook', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('stops orphan recovery before a fast route DOM swap can replay old bids', async () => { + document.body.innerHTML = '
'; + const definedDivs: string[] = []; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + refresh: vi.fn(), + addEventListener: vi.fn(), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn((_path: string, _sizes: unknown, divId: string) => { + definedDivs.push(divId); + return { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue(divId), + getTargeting: vi.fn().mockReturnValue([]), + }; + }), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + display: vi.fn(), + destroySlots: vi.fn(), + }; + // Keep page-bids slower than the orphan observer's 250 ms debounce. + fetchStub.mockReturnValue(new Promise(() => {})); + + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adSlots = [ + { + id: 'ad-header-0', + gam_unit_path: '/123/header', + div_id: 'ad-header-0', + formats: [[728, 90]], + targeting: {}, + }, + ]; + ts.bids = { 'ad-header-0': { hb_adid: 'old-route-ad' } }; + ts.adInit!(); + expect(definedDivs).toEqual(['ad-header-0-_R_old_']); + + history.pushState({}, '', '/new-route'); + document.body.innerHTML = '
'; + await new Promise((resolve) => setTimeout(resolve, 350)); + + // The pending old-route watcher was disconnected synchronously when + // navigation began, so it never rebound or re-requested the old auction. + expect(definedDivs).toEqual(['ad-header-0-_R_old_']); + }); + it('leaves slots and bids untouched on a non-OK response', async () => { fetchStub.mockResolvedValue({ ok: false, status: 500 }); const { installSpaAuctionHook } = await importGptModule(); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 00cf99dd0..64cc6ddf8 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -100,8 +100,11 @@ import { auctionBidsToPrebidBids, installPrebidNpm, installRefreshHandler, + installPrebidRenderTrace, + recordPrebidAdRender, } from '../../../src/integrations/prebid/index'; import type { AuctionBid } from '../../../src/core/auction'; +import type { TsjsApi } from '../../../src/core/types'; import { log } from '../../../src/core/log'; describe('prebid/collectBidders', () => { @@ -266,6 +269,184 @@ describe('prebid/auctionBidsToPrebidBids', () => { expect(result[0].requestId).toBe('req-a'); expect(result[1].requestId).toBe('req-b'); }); + + it('forwards the server-side trace tuple into Prebid meta', () => { + const auctionBids: AuctionBid[] = [ + { + impid: 'div-gpt-1', + adm: '
Ad
', + price: 1.0, + width: 300, + height: 250, + seat: 'kargo', + creativeId: 'KM-CREA-1', + adomain: ['kargo.com'], + auctionId: 'ts-auction-xyz', + bidId: 'bid-abc-1', + admHash: 'a1b2c3d4e5f60718', + }, + ]; + + const result = auctionBidsToPrebidBids(auctionBids, []); + + expect(result[0].meta.tsAuctionId).toBe('ts-auction-xyz'); + expect(result[0].meta.tsAdmHash).toBe('a1b2c3d4e5f60718'); + // The bid's own OpenRTB id, distinct from the advertiser creative id. + expect(result[0].meta.tsBidId).toBe('bid-abc-1'); + expect(result[0].creativeId).toBe('KM-CREA-1'); + }); +}); + +describe('prebid/recordPrebidAdRender', () => { + beforeEach(() => { + delete (window as { tsjs?: TsjsApi }).tsjs; + document.body.innerHTML = ''; + }); + afterEach(() => { + delete (window as { tsjs?: TsjsApi }).tsjs; + document.body.innerHTML = ''; + }); + + it('records an auction-path render for a server-side bid', () => { + document.body.innerHTML = '
'; + const record = recordPrebidAdRender( + { + adUnitCode: 'ad-header-0-_R_x_', + bidderCode: 'kargo', + creativeId: 'KM-CREA-1', + meta: { + tsAuctionId: '265dcedd-aa0a', + tsBidId: 'bid-abc-1', + tsAdmHash: 'f68044ca9f68c88c', + }, + }, + 'succeeded' + ); + + expect(record).toBeDefined(); + expect(record).toEqual( + expect.objectContaining({ + slotId: 'ad-header-0-_R_x_', + path: 'auction', + rendered: true, + injected: true, + auctionId: '265dcedd-aa0a', + bidId: 'bid-abc-1', + admHash: 'f68044ca9f68c88c', + bidder: 'kargo', + creativeId: 'KM-CREA-1', + servedFrom: 'prebid', + elementId: 'ad-header-0-_R_x_', + }) + ); + // Written into the shared registry the panel reads. + expect((window as { tsjs?: TsjsApi }).tsjs?.renders?.['ad-header-0-_R_x_']).toBeDefined(); + // The bid id must reach the DOM as its own attribute, never folded into + // data-ts-ad-id. + const el = document.getElementById('ad-header-0-_R_x_')!; + expect(el.getAttribute('data-ts-bid-id')).toBe('bid-abc-1'); + }); + + it('records a failed render as unconfirmed, not as a green render', () => { + document.body.innerHTML = '
'; + const record = recordPrebidAdRender( + { + adUnitCode: 'ad-header-0', + bidderCode: 'kargo', + meta: { tsAuctionId: '265dcedd-aa0a' }, + }, + 'failed' + ); + + expect(record).toEqual( + expect.objectContaining({ rendered: false, injected: false, visible: false }) + ); + }); + + it('skips a bid without the server-side trace tuple (client-side bidder)', () => { + const record = recordPrebidAdRender( + { + adUnitCode: 'ad-header-0', + bidderCode: 'appnexus', + meta: { advertiserDomains: ['x.com'] }, + }, + 'succeeded' + ); + expect(record).toBeUndefined(); + expect((window as { tsjs?: TsjsApi }).tsjs?.renders).toBeUndefined(); + }); + + it('skips a bid with no adUnitCode', () => { + expect(recordPrebidAdRender({ meta: { tsAuctionId: 'x' } }, 'succeeded')).toBeUndefined(); + expect(recordPrebidAdRender(undefined, 'succeeded')).toBeUndefined(); + }); +}); + +describe('prebid/installPrebidRenderTrace', () => { + beforeEach(() => { + delete (window as { tsjs?: TsjsApi }).tsjs; + document.body.innerHTML = ''; + mockOnEvent.mockReset(); + delete (mockPbjs as { __tsRenderTraceInstalled?: boolean }).__tsRenderTraceInstalled; + }); + afterEach(() => { + delete (window as { tsjs?: TsjsApi }).tsjs; + document.body.innerHTML = ''; + }); + + it('confirms renders from adRenderSucceeded, never from bidWon', () => { + document.body.innerHTML = '
'; + installPrebidRenderTrace(); + + const events = mockOnEvent.mock.calls.map(([name]) => name); + expect(events).toEqual(['adRenderSucceeded', 'adRenderFailed']); + // bidWon fires when a bid is marked the winner — before the renderer runs, + // and so before the render can fail. Confirming on it would show a green + // render for a creative that never reached the page. + expect(events).not.toContain('bidWon'); + + const handlers = Object.fromEntries(mockOnEvent.mock.calls) as Record< + string, + (event: unknown) => void + >; + handlers['adRenderSucceeded']({ + bid: { + adUnitCode: 'ad-header-0', + bidderCode: 'kargo', + meta: { tsAuctionId: 'auction-success', tsBidId: 'bid-success' }, + }, + }); + expect((window as { tsjs?: TsjsApi }).tsjs?.renders?.['ad-header-0']).toEqual( + expect.objectContaining({ rendered: true, injected: true, bidId: 'bid-success' }) + ); + }); + + it('does not produce a confirmed record when the render fails after the win', () => { + document.body.innerHTML = '
'; + installPrebidRenderTrace(); + + const handlers = Object.fromEntries(mockOnEvent.mock.calls) as Record< + string, + (event: unknown) => void + >; + const bid = { + adUnitCode: 'ad-header-0', + bidderCode: 'kargo', + meta: { tsAuctionId: '265dcedd-aa0a' }, + }; + + // Prebid marks the bid as won, then its renderer fails asynchronously. + handlers['adRenderFailed']({ reason: 'exception', message: 'boom', bid }); + + const record = (window as { tsjs?: TsjsApi }).tsjs?.renders?.['ad-header-0']; + expect(record).toEqual( + expect.objectContaining({ rendered: false, injected: false, visible: false }) + ); + // No green badge and no confirmed-render attributes on the slot. + const el = document.getElementById('ad-header-0')!; + expect(el.getAttribute('data-ts-rendered')).toBe('false'); + expect(el.getAttribute('data-ts-injected')).toBe('false'); + }); }); describe('prebid/installPrebidNpm', () => { diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 9e8c70b24..f97e6df97 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -161,14 +161,14 @@ Core publisher settings for domain, origin, and proxy configuration. ### `[publisher]` -| Field | Type | Required | Description | -| ----------------------------- | ------- | -------- | --------------------------------------------------------------------------------------- | -| `domain` | String | Yes | Publisher's apex domain name | -| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | -| `origin_url` | String | Yes | Full URL of publisher origin server | -| `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | -| `proxy_secret` | String | Yes | Secret key for encrypting/signing proxy URLs | -| `max_buffered_body_bytes` | Integer | No | Max bytes buffered when a publisher response is post-processed in full (default 16 MiB) | +| Field | Type | Required | Description | +| ----------------------------- | ------- | -------- | --------------------------------------------------------------------------- | +| `domain` | String | Yes | Publisher's apex domain name | +| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | +| `origin_url` | String | Yes | Full URL of publisher origin server | +| `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | +| `proxy_secret` | String | Yes | Secret key for encrypting/signing proxy URLs | +| `max_buffered_body_bytes` | Integer | No | Buffered-body cap / Fastly stream raw+decoded byte ceiling (default 16 MiB) | > **Note:** EC cookies (`ts-ec`) derive their domain automatically as `.{domain}` and > do not use `cookie_domain`. The `cookie_domain` field is used by other cookie helpers. @@ -300,29 +300,36 @@ Changing `proxy_secret` invalidates all existing signed URLs. Plan rotations car #### `max_buffered_body_bytes` -**Purpose**: Upper bound on the in-memory buffer used when a publisher origin -response must be processed in full (HTML rewriting and integration injection) -instead of streamed. +**Purpose**: Upper bound on how much of a publisher origin body the rewrite +pipeline holds in memory — the post-rewrite output buffer on buffered adapters, +and the per-stream raw/decoded byte ceiling on the Fastly streaming path. **Usage**: -- Caps the _decoded, post-rewrite_ output buffer for any buffered publisher - response on both the legacy and EdgeZero paths. -- Exceeding the cap fails the response (mapped to a 5xx proxy error) rather than - allocating past the limit, preventing Wasm-heap exhaustion on highly - compressible documents. +- On **buffered adapters** (Axum, Cloudflare, Spin) it caps the _decoded, + post-rewrite_ output buffer for a publisher response processed in full. +- On the **Fastly streaming path** the origin body is preserved as a stream, so + the same value caps the stream twice over: the cumulative _raw_ (still + compressed) bytes pulled from origin, and the cumulative _decoded_ bytes + emitted by the decompressor. The decoded cap is enforced _during_ + decompression, so a decompression bomb is rejected before its expansion is + materialized rather than after. -**Default**: `16777216` (16 MiB). +**Behavior when exceeded**: -**Effective Fastly limit**: On Fastly the practical ceiling for a publisher page -is lower. The platform HTTP client rejects any origin response whose raw (still -compressed) body exceeds **10 MiB** before this buffer is filled, so raising the -value only helps highly compressible pages whose decoded size exceeds 16 MiB -while their compressed origin body stays under 10 MiB. Raising it above ~10 MiB -does not lift the platform cap for uncompressed pages. +- On **buffered adapters** the response fails before any bytes are committed. +- On the **streaming path** the response headers are already committed when + either cap trips, so the body is **truncated mid-stream** and the error is + logged — the client receives a short (incomplete) body rather than a `5xx`. + Size the cap above your largest expected decoded page so legitimate responses + are never truncated. + +**Default**: `16777216` (16 MiB). On the Fastly streaming path this is now the +sole ceiling: origin bodies are streamed rather than materialized in full, so +the previous ~10 MiB raw-body limit no longer applies. **Minimum**: Must be at least `1`. A value of `0` is rejected at startup because -a zero-byte cap fails every non-empty buffered response. +a zero-byte cap fails every non-empty publisher response. **Environment Override**: diff --git a/docs/superpowers/plans/2026-07-08-true-origin-streaming-fastly.md b/docs/superpowers/plans/2026-07-08-true-origin-streaming-fastly.md index bc1983a97..677e68c85 100644 --- a/docs/superpowers/plans/2026-07-08-true-origin-streaming-fastly.md +++ b/docs/superpowers/plans/2026-07-08-true-origin-streaming-fastly.md @@ -22,6 +22,15 @@ In scope: Out of scope: +- HTML post-processor configs (the `nextjs` integration). When any + `IntegrationHtmlPostProcessor` is registered, `HtmlWithPostProcessing` + accumulates the full rewritten document and runs post-processors at origin + EOF, so the lazy stream emits no body bytes until the whole origin transfer + completes — even for pages where `should_process()` would return `false`. + Headers still commit early, but first byte/FCP tracks origin EOF for that + configuration; #849's objective is unmet there. The eventual fix is an + up-front or streaming `should_process` gate so non-RSC pages skip + accumulation entirely (follow-up issue). - Cloudflare origin streaming. Current adapter rejects `PlatformHttpRequest::stream_response`. - Spin streaming. Current adapter and upstream EdgeZero Spin conversion are buffered/blocking issues. - Axum client streaming. Axum is dev-only and has `LocalBoxStream`/`Send` constraints. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index ef3edc2af..8615e652f 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -157,6 +157,15 @@ ja4_endpoint_enabled = false # in production. auction_html_comment = false +# Expose GET /_ts/trace, which toggles the `ts-trace` cookie and redirects to /. +# While the cookie is set, the TSJS overlay draws a floating panel summarising +# every traced ad slot (render path, bidder, and GAM/injected/visible state) +# plus a confirmation badge on each genuinely-rendered creative. It only +# surfaces data already present on window.tsjs, so it leaks nothing new — but +# it is off by default so the toggle route is not live on deployments that +# never asked for it. Enable only for render-verification debugging. +# trace_route_enabled = false + [creative_opportunities] gam_network_id = "123456789" # FCP is not affected by this value — body content above has already