diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index f2df61744..8fccaf1d0 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -238,6 +238,9 @@ mod tests { use super::*; + const TEST_USER_AGENT: &str = + "FictionalBrowser/123.4 (FictionalOS 10.2; FictionalDevice) ExampleRenderer/567.8"; + struct NoopConfigStore; impl PlatformConfigStore for NoopConfigStore { @@ -399,6 +402,7 @@ mod tests { region: None, is_mobile: 0, is_known_browser: 1, + user_agent: Some(TEST_USER_AGENT.to_owned()), gdpr_applies: 0, consent_present: 0, terminal_status: Some("completed".to_owned()), @@ -523,11 +527,11 @@ mod tests { header_value(&requests[0].headers, header::AUTHORIZATION.as_str()), Some("Bearer append-token") ); + let body = std::str::from_utf8(&requests[0].body).expect("should record utf8 ndjson body"); + assert!(body.ends_with('\n'), "should send newline-delimited JSON"); assert!( - std::str::from_utf8(&requests[0].body) - .expect("should record utf8 ndjson body") - .ends_with('\n'), - "should send newline-delimited JSON" + body.contains(TEST_USER_AGENT), + "should send the complete user agent to Tinybird" ); assert_eq!( *http_client diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index b3e049eaf..d6076055d 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -1,7 +1,7 @@ //! Auction telemetry row construction and sink abstraction. //! -//! Core owns the privacy-preserving auction observation model and pure row -//! builder. Platform adapters provide the concrete sink implementation. +//! Core owns the auction observation model and pure row builder. Platform +//! adapters provide the concrete sink implementation. use std::collections::HashSet; use std::time::Instant; @@ -19,6 +19,9 @@ use crate::platform::RuntimeServices; const MAX_PAGE_PATH_BYTES: usize = 256; const DYNAMIC_SEGMENT_REPLACEMENT: &str = ":id"; +#[cfg(test)] +const TEST_USER_AGENT: &str = + "FictionalBrowser/123.4 (FictionalOS 10.2; FictionalDevice) ExampleRenderer/567.8"; /// Source path that initiated an auction candidate. #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -91,7 +94,7 @@ impl AbandonedProviderCall { } } -/// Privacy-preserving context shared by all rows in one auction observation. +/// Context shared by all rows in one auction observation. #[derive(Debug, Clone)] pub struct AuctionObservationContext { /// Fresh telemetry UUID, independent of EC and internal auction IDs. @@ -110,6 +113,8 @@ pub struct AuctionObservationContext { pub is_mobile: u8, /// `0` = bot, `1` = browser, `2` = unknown. pub is_known_browser: u8, + /// Complete User-Agent supplied by the auction request, when present. + pub user_agent: Option, /// Whether GDPR applies. pub gdpr_applies: bool, /// Whether any consent signal was present. @@ -134,11 +139,16 @@ impl AuctionObservationContext { .and_then(|page_url| url::Url::parse(page_url).ok()) .map(|url| url.path().to_owned()) .unwrap_or_else(|| "/".to_owned()); + let user_agent = request + .device + .as_ref() + .and_then(|device| device.user_agent.as_deref()); Self::from_parts( auction_source, &request.publisher.domain, &raw_path, request.slots.len(), + user_agent, ec_context, ) } @@ -150,6 +160,7 @@ impl AuctionObservationContext { publisher_domain: &str, raw_page_path: &str, slot_count: usize, + user_agent: Option<&str>, ec_context: &EcContext, ) -> Self { let device = ec_context.device_signals(); @@ -172,6 +183,7 @@ impl AuctionObservationContext { Some(false) => 0, None => 2, }, + user_agent: user_agent.map(str::to_owned), gdpr_applies: consent.gdpr_applies, consent_present: !consent.is_empty(), slot_count, @@ -196,6 +208,7 @@ impl AuctionObservationContext { region: Some("CA".to_owned()), is_mobile: 0, is_known_browser: 1, + user_agent: Some(TEST_USER_AGENT.to_owned()), gdpr_applies: false, consent_present: false, slot_count, @@ -282,6 +295,8 @@ pub struct AuctionEventRow { pub is_mobile: u8, /// `0` = bot, `1` = browser, `2` = unknown. pub is_known_browser: u8, + /// Complete User-Agent supplied by the auction request, when present. + pub user_agent: Option, /// `0` or `1`. pub gdpr_applies: u8, /// `0` or `1`. @@ -341,6 +356,7 @@ impl AuctionEventRow { region: observation.region.clone(), is_mobile: observation.is_mobile, is_known_browser: observation.is_known_browser, + user_agent: observation.user_agent.clone(), gdpr_applies: u8::from(observation.gdpr_applies), consent_present: u8::from(observation.consent_present), terminal_status: None, @@ -933,7 +949,7 @@ mod tests { use serde_json::json; - use crate::auction::types::{AdFormat, AdSlot, PublisherInfo, UserInfo}; + use crate::auction::types::{AdFormat, AdSlot, DeviceInfo, PublisherInfo, UserInfo}; use super::*; @@ -1005,6 +1021,52 @@ mod tests { ); } + #[test] + fn observation_sources_preserve_complete_user_agent() { + let ec_context = + EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); + let mut request = test_request("request-id"); + request.device = Some(DeviceInfo { + user_agent: Some(TEST_USER_AGENT.to_owned()), + ip: None, + geo: None, + }); + + let from_request = AuctionObservationContext::from_auction_request( + AuctionSource::AuctionApi, + &request, + &ec_context, + ); + let from_parts = AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "test-publisher.example", + "/article", + 1, + Some(TEST_USER_AGENT), + &ec_context, + ); + let without_user_agent = AuctionObservationContext::from_auction_request( + AuctionSource::AuctionApi, + &test_request("request-without-user-agent"), + &ec_context, + ); + + assert_eq!( + from_request.user_agent.as_deref(), + Some(TEST_USER_AGENT), + "should preserve the complete user agent from an auction request" + ); + assert_eq!( + from_parts.user_agent.as_deref(), + Some(TEST_USER_AGENT), + "should preserve the complete user agent from publisher request parts" + ); + assert_eq!( + without_user_agent.user_agent, None, + "should omit a missing user agent" + ); + } + #[test] fn normalize_page_path_strips_query_and_redacts_dynamic_segments() { assert_eq!( @@ -1065,6 +1127,11 @@ mod tests { ); let rows = batch.rows(); + assert!( + rows.iter() + .all(|row| row.user_agent.as_deref() == Some(TEST_USER_AGENT)), + "should copy the complete user agent to every row kind" + ); assert_eq!( rows.iter() .filter(|row| row.event_kind == "summary") @@ -1249,6 +1316,10 @@ mod tests { .expect("should serialize ndjson"); assert!(body.ends_with('\n'), "should end each row with newline"); + assert!( + body.contains(TEST_USER_AGENT), + "should preserve the complete user agent in serialized rows" + ); for line in body.lines() { let parsed: serde_json::Value = serde_json::from_str(line).expect("should parse row"); assert_eq!(parsed["event_kind"], "summary"); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 40bb7ea27..617e944fc 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -4214,11 +4214,16 @@ pub async fn handle_publisher_request( // trusted-server edge host, so using it here would attribute navigation // rows to the edge/staging domain while `/auction` rows (built from // `AuctionRequest::publisher.domain`) use the configured domain. + let user_agent = req + .headers() + .get("user-agent") + .and_then(|value| value.to_str().ok()); let observation = AuctionObservationContext::from_parts( AuctionSource::InitialNavigation, &settings.publisher.domain, &request_path, matched_slots.len(), + user_agent, ec_context, ); @@ -4233,9 +4238,7 @@ pub async fn handle_publisher_request( &consent_context, &request_info, &settings.publisher.domain, - req.headers() - .get("user-agent") - .and_then(|v| v.to_str().ok()), + user_agent, ); apply_auction_eids_and_device( &mut auction_request, @@ -6453,11 +6456,16 @@ pub async fn handle_page_bids( } else { // Same publisher identity as the outbound bid request — see the // matching note on the initial-navigation observation above. + let user_agent = req + .headers() + .get("user-agent") + .and_then(|value| value.to_str().ok()); let observation = AuctionObservationContext::from_parts( AuctionSource::SpaNavigation, &settings.publisher.domain, &path_param, matched_slots.len(), + user_agent, ec_context, ); if ad_stack_enabled && !is_bot && !is_prefetch { @@ -6471,9 +6479,7 @@ pub async fn handle_page_bids( consent_context, &request_info, &settings.publisher.domain, - req.headers() - .get("user-agent") - .and_then(|v| v.to_str().ok()), + user_agent, ); apply_auction_eids_and_device( &mut auction_request, @@ -17031,6 +17037,7 @@ mod tests { "proxy.example.com", "/article", 1, + Some("FictionalBrowser/123.4"), &ec_context, )), auction_request: Some(test_auction_request()), diff --git a/docs/superpowers/plans/2026-06-23-auction-telemetry-direct-tinybird.md b/docs/superpowers/plans/2026-06-23-auction-telemetry-direct-tinybird.md index 5cf0ecfa7..8057d87e2 100644 --- a/docs/superpowers/plans/2026-06-23-auction-telemetry-direct-tinybird.md +++ b/docs/superpowers/plans/2026-06-23-auction-telemetry-direct-tinybird.md @@ -34,7 +34,8 @@ The implementation must: - cover `POST /auction`, `GET /__ts/page-bids`, and initial-navigation SSAT dispatch/collect; - emit exactly one summary row per auction candidate per successful Compute execution path; - batch all rows for one auction observation into one Tinybird Events API POST; -- omit EC ID, internal `AuctionRequest.id`, IP, raw user agent, full URL, query strings, and fragments; +- omit EC ID, internal `AuctionRequest.id`, IP, full URL, query strings, and fragments; +- preserve the complete User-Agent for downstream data-sync classification; - generate a fresh random telemetry UUID unrelated to EC or request IDs; - emit provider-call rows for provider launch, parse, transport, timeout/no-response, no-bid, success, and abandoned outcomes where observable; - avoid invented seat-level no-bids; @@ -573,12 +574,12 @@ Do this only if requested for a future implementation PR; auction telemetry alon - Launch, parse, transport, timeout, no-bid, success, abandoned statuses map correctly. - Mediated winners produce exactly one canonical winning bid row per slot. - Telemetry UUID is fresh and independent of `AuctionRequest.id`/EC. -- Privacy fields are absent from serialized rows: +- Sensitive identity fields are absent from serialized rows: - no EC ID; - no internal request ID; - no IP; - - no raw UA; - no full URL/query/fragment. +- The complete User-Agent is preserved without edge-side classification. - Page path normalization is bounded and redacts dynamic segments. - NDJSON serialization emits valid single-line JSON rows and a trailing newline. diff --git a/docs/superpowers/specs/2026-06-22-auction-prebid-metrics-tinybird-grafana-design.md b/docs/superpowers/specs/2026-06-22-auction-prebid-metrics-tinybird-grafana-design.md index 09d5a70e8..0214e5251 100644 --- a/docs/superpowers/specs/2026-06-22-auction-prebid-metrics-tinybird-grafana-design.md +++ b/docs/superpowers/specs/2026-06-22-auction-prebid-metrics-tinybird-grafana-design.md @@ -307,7 +307,8 @@ winner row. Never mark both an original and mediator copy as the same win. Emission is not gated by consent. `gdpr_applies` comes from `ConsentContext.gdpr_applies`; `consent_present` is `!ConsentContext::is_empty()` because an `EcContext` always contains a consent context, even when no signal was -supplied. +supplied. The complete request User-Agent is emitted for downstream data-sync +classification, including on consent-denied observations. #### `auction_events_raw` row schema @@ -325,6 +326,7 @@ All row kinds share these columns: | `region` | Nullable(String) | coarse geo | | `is_mobile` | UInt8 | 0=desktop, 1=mobile, 2=unknown | | `is_known_browser` | UInt8 | 0=bot, 1=browser, 2=unknown | +| `user_agent` | Nullable(String) | complete request User-Agent for data syncs | | `gdpr_applies` | UInt8 | 0/1 | | `consent_present` | UInt8 | 0/1 | @@ -364,11 +366,13 @@ Bid fields: | `ad_domain` | Nullable(String) | advertiser domain, optional | | `ad_id` | Nullable(String) | creative ID, optional | -Privacy note: `auction_id` is generated independently for telemetry. No EC ID, -internal auction request ID, full URL, IP, or raw user-agent string is emitted. -Page paths use the same bounded route-normalization principle as access logs so -a dynamic path segment cannot become a per-user identifier. Geo remains at -country/region granularity. +Data contract note: `auction_id` is generated independently for telemetry. No +EC ID, internal auction request ID, full URL, or IP is emitted. The complete +User-Agent is intentionally retained in the raw datasource so downstream data +syncs, rather than edge code, own browser classification. Page paths use the +same bounded route-normalization principle as access logs so a dynamic path +segment cannot become a per-user identifier. Geo remains at country/region +granularity. Device signals note: both `is_mobile` (0/1/2) and the bot-vs-browser bit come from the adapter's already-derived device signals. Phase 1 snapshots that struct @@ -605,7 +609,8 @@ the direct-ingest architecture. - Provider launch, parse, transport, timeout, no-bid, and success outcomes are represented consistently on synchronous and split-phase paths. - Telemetry auction IDs are fresh random UUIDs unrelated to internal request IDs - and EC values. No EC ID, full URL, IP, or raw user-agent string is emitted. + and EC values. No EC ID, full URL, or IP is emitted. The complete User-Agent + is retained for downstream data-sync classification. - The Fastly adapter sends auction rows to Tinybird through a direct asynchronous backend POST using a Secret Store token scoped to APPEND on `auction_events_raw`; it does not configure or require Fastly real-time diff --git a/tinybird/datasources/auction_events_raw.datasource b/tinybird/datasources/auction_events_raw.datasource index d62f8ae5d..97e8251bb 100644 --- a/tinybird/datasources/auction_events_raw.datasource +++ b/tinybird/datasources/auction_events_raw.datasource @@ -12,6 +12,7 @@ SCHEMA > `region` Nullable(String), `is_mobile` UInt8, `is_known_browser` UInt8, + `user_agent` Nullable(String), `gdpr_applies` UInt8, `consent_present` UInt8, `terminal_status` LowCardinality(Nullable(String)),